diff --git a/nextflow/modules/fecal_boli.nf b/nextflow/modules/fecal_boli.nf index 503a759d..b1236a8a 100644 --- a/nextflow/modules/fecal_boli.nf +++ b/nextflow/modules/fecal_boli.nf @@ -1,3 +1,18 @@ +/** + * This module contains process definitions for fecal boli prediction and feature calculations. + */ + + /** + * Predicts fecal boli in the video at minute intervals. + * + * @param tuple + * - video_file The input video file. + * - in_pose The input pose file. + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with fecal boli predictions. + */ process PREDICT_FECAL_BOLI { label "gpu" label "tracking" @@ -16,6 +31,15 @@ process PREDICT_FECAL_BOLI { """ } +/** + * Extracts fecal boli features from the pose data. + * + * @param tuple + * - video_file The input video file. + * - in_pose The input pose file. + * + * @return fecal_boli The generated fecal boli feature CSV file. + */ process EXTRACT_FECAL_BOLI_BINS { label "tracking" label "r_fboli_extract" diff --git a/nextflow/modules/flexibility.nf b/nextflow/modules/flexibility.nf index bbb3387f..3d73cfc2 100644 --- a/nextflow/modules/flexibility.nf +++ b/nextflow/modules/flexibility.nf @@ -1,3 +1,21 @@ +/** + * This module contains process definitions for generating flexibility metrics. + * These metrics were published in https://doi.org/10.1038/s43587-022-00266-0 + */ + +/** + * Generates flexibility index metrics from pose data. + * + * @param tuple + * - video_file The input video file. + * - pose_file The input pose file. + * + * @return tuple files + * - Path to the generated angles CSV file. + * - Path to the generated dAC CSV file. + * - Path to the generated dB CSV file. + * - Path to the generated flexdexraw CSV file. + */ process GENERATE_FLEXIBILITY_INDEX { label "frailty" label "cpu" @@ -32,6 +50,16 @@ process GENERATE_FLEXIBILITY_INDEX { """ } +/** + * Generates rear paw width metrics from pose data. + * + * @param tuple + * - video_file The input video file. + * - pose_file The input pose file. + * + * @return rearpaw Path to the generated rear paw width CSV file. + * @return rearpawsave Path to the generated rear paw save CSV file. + */ process GENERATE_REAR_PAW_WIDTH { label "frailty" label "cpu" diff --git a/nextflow/modules/gait.nf b/nextflow/modules/gait.nf index 2e1f1300..c44cbd4a 100644 --- a/nextflow/modules/gait.nf +++ b/nextflow/modules/gait.nf @@ -1,3 +1,16 @@ +/** + * This module contains process definitions for gait analysis. + */ + +/** + * Generates gait statistics from pose data. + * + * @param tuple + * - video_file The input video file. + * - pose_file The input pose file. + * + * @return gait_file The generated gait statistics file. + */ process GENERATE_GAIT_H5 { label "gait" label "cpu" @@ -24,7 +37,15 @@ process GENERATE_GAIT_H5 { """ } - +/** + * Generates gait bin files from gait statistics. + * + * @param tuple + * - gait_file The input gait statistics file. + * - speed_bin The speed bin size for generating gait bins. + * + * @return gait_bin_csv The generated gait bin CSV file. + */ process GENERATE_GAIT_BIN { label "gait" label "cpu" diff --git a/nextflow/modules/jabs_classifiers.nf b/nextflow/modules/jabs_classifiers.nf index af393c9e..c29607f4 100644 --- a/nextflow/modules/jabs_classifiers.nf +++ b/nextflow/modules/jabs_classifiers.nf @@ -1,3 +1,18 @@ +/** + * This module contains process definitions related to JABS classifiers. + */ + +/** + * Calculates features for JABS classifiers. + * + * @param tuple + * - video_file The input video file. + * - in_pose The input pose file. + * + * @return tuple files + * - Path to the original pose file. + * - Path to the generated feature cache directory. + */ process GENERATE_FEATURE_CACHE { // This process will correct pose pathing to a v6 file label "jabs_classify" @@ -23,6 +38,20 @@ process GENERATE_FEATURE_CACHE { """ } +/** + * Predicts behaviors using JABS classifiers. + * + * @param tuple + * - in_pose The input pose file. + * - feature_cache The directory containing the generated features. + * @param classifiers A map of classifiers: + * - : classifier parameter maps (not used by this process) + * + * @return tuple files + * - Path to the original pose file. + * - Path to the feature cache directory. + * - Path to the generated behavior file. All behavior predictions are stored in a single file. + */ process PREDICT_CLASSIFIERS { label "jabs_classify" label "cpu" @@ -46,6 +75,22 @@ process PREDICT_CLASSIFIERS { """ } +/** + * Generates behavior tables from pose file, feature file, and prediction file. + * + * @param tuple + * - in_pose The input pose file. + * - feature_cache The directory containing the generated features. + * - behavior_files The behavior prediction file. + * @param classifiers A map of classifiers: + * - : classifier parameter maps: + * - stitch_value: the gap size for stitching behavior bouts + * - filter_value: the minimum length for behavior bouts + * + * @return tuple files + * - Path to the generated behavior bout file. + * - Path to the generated behavior summary file. + */ process GENERATE_BEHAVIOR_TABLES { label "jabs_postprocess" label "cpu" @@ -53,18 +98,30 @@ process GENERATE_BEHAVIOR_TABLES { input: tuple path(in_pose), path(feature_cache), path(behavior_files) - val classifier + val classifiers output: tuple path("${in_pose.baseName}*_bouts.csv"), path("${in_pose.baseName}*_summaries.csv"), emit: files script: """ - behavior_command="--behavior ${classifier.collect { entry -> "$entry.key --stitch_gap $entry.value.stitch_value --min_bout_length $entry.value.filter_value" }.join(' --behavior ')}" + behavior_command="--behavior ${classifiers.collect { entry -> "$entry.key --stitch_gap $entry.value.stitch_value --min_bout_length $entry.value.filter_value" }.join(' --behavior ')}" python3 /JABS-postprocess/generate_behavior_tables.py --project_folder . --feature_folder . --out_prefix ${in_pose.baseName} --out_bin_size 5 \${behavior_command} """ } +/** + * Generates heuristic classifier predictions. + * + * @param tuple + * - in_pose The input pose file. + * - feature_cache The directory containing the generated features. + * @param heuristic_classifiers A list of heuristic classifier configuration files. + * + * @return tuple files + * - Path to the generated bout file. + * - Path to the generated summary file. + */ process PREDICT_HEURISTICS { label "jabs_postprocess" label "cpu" @@ -87,6 +144,14 @@ process PREDICT_HEURISTICS { """ } +/** + * Converts a behavior summary table to features. + * + * @param in_summary_table The input behavior summary table. + * @param bin_size The bin size for feature extraction. + * + * @return features The generated feature file. + */ process BEHAVIOR_TABLE_TO_FEATURES { label "jabs_table_convert" label "r_jabs_table_convert" diff --git a/nextflow/modules/manual_correction.nf b/nextflow/modules/manual_correction.nf index 745f51b2..6ee6c946 100644 --- a/nextflow/modules/manual_correction.nf +++ b/nextflow/modules/manual_correction.nf @@ -1,3 +1,17 @@ +/** + * This module contains process definitions related to manually correcting pose data using SLEAP. + */ + +/** + * Extracts a frame from a video as a png image. + * + * @param tuple + * - video The input video file. + * - pose_file The input pose file (not used in this process). + * @param frame_index The index of the frame to extract. + * + * @return frame A png image of the extracted frame. + */ process EXTRACT_VIDEO_FRAME { label "sleap" @@ -14,6 +28,13 @@ process EXTRACT_VIDEO_FRAME { """ } +/** + * Adds a set of frames to a new SLEAP project file for arena corner annotation. + * + * @param video_frames A list of png images to be added to the SLEAP project. + * + * @return sleap_file The generated SLEAP project file containing the frames and arena corner skeleton for annotation. + */ process ADD_EXAMPLES_TO_SLEAP { label "sleap" @@ -48,6 +69,14 @@ process ADD_EXAMPLES_TO_SLEAP { """ } +/** + * Integrates SLEAP corner annotations into a pose file. + * + * @param pose_file The input pose file to be corrected. + * @param sleap_file The SLEAP project file containing the corner annotations. + * + * @return pose_file The corrected pose file with integrated corner annotations. + */ process INTEGRATE_SLEAP_CORNER_ANNOTATIONS { label "sleap_io" diff --git a/nextflow/modules/multi_mouse.nf b/nextflow/modules/multi_mouse.nf index f8c155b9..4109b2c7 100644 --- a/nextflow/modules/multi_mouse.nf +++ b/nextflow/modules/multi_mouse.nf @@ -1,3 +1,14 @@ +/** + * Predicts multi-mouse segmentation. + * + * @param tuple + * - video_file The input video file + * - in_pose The input pose file + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with multi-mouse segmentation predicted. + */ process PREDICT_MULTI_MOUSE_SEGMENTATION { label "gpu" label "tracking" @@ -15,6 +26,17 @@ process PREDICT_MULTI_MOUSE_SEGMENTATION { """ } +/** + * Predicts multi-mouse keypoints. + * + * @param tuple + * - video_file The input video file + * - in_pose The input pose file + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with multi-mouse keypoints predicted. + */ process PREDICT_MULTI_MOUSE_KEYPOINTS { label "gpu" label "tracking" @@ -32,6 +54,17 @@ process PREDICT_MULTI_MOUSE_KEYPOINTS { """ } +/** + * Predicts multi-mouse identity. + * + * @param tuple + * - video_file The input video file + * - in_pose The input pose file + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with multi-mouse identity predicted. + */ process PREDICT_MULTI_MOUSE_IDENTITY { label "gpu" label "tracking" @@ -49,6 +82,18 @@ process PREDICT_MULTI_MOUSE_IDENTITY { """ } +/** + * Generates multi-mouse tracklets from the pose, segmentaiton, and identity data. + * + * @param tuple + * - video_file The input video file + * - in_pose The input pose file + * - num_animals The number of animals to generate tracklets for + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with multi-mouse tracklets generated. + */ process GENERATE_MULTI_MOUSE_TRACKLETS { label "cpu" label "tracking" diff --git a/nextflow/modules/remote_io.nf b/nextflow/modules/remote_io.nf index a0a4bae0..92cbf883 100644 --- a/nextflow/modules/remote_io.nf +++ b/nextflow/modules/remote_io.nf @@ -1,3 +1,14 @@ +/** + * This module contains process definitions for remote I/O operations. + */ + +/** + * Checks if the user has a valid Globus authentication session. + * + * @param globus_endpoint The Globus endpoint to check authentication against + * + * @exception If the user is not authenticated, the process will exit with an error. + */ process CHECK_GLOBUS_AUTH { label "globus" @@ -21,6 +32,14 @@ process CHECK_GLOBUS_AUTH { // But this needs to be parsed and compared with the endpoint expiration } +/** + * Filters a list of video files to only those that do not have corresponding processed pose files on Globus. + * + * @param globus_endpoint The Globus endpoint where processed pose files are stored + * @param test_files A file containing a list of video files to check + * + * @return unprocessed_files A file containing a list of video files that do not have a corresponding pose file. + */ process FILTER_UNPROCESSED_GLOBUS { label "globus" @@ -44,6 +63,14 @@ process FILTER_UNPROCESSED_GLOBUS { """ } +/** + * Filters a list of video files to only those that do not have corresponding processed pose files on Dropbox. + * + * @param dropbox_prefix The rclone remote prefix where processed pose files are stored + * @param test_files A file containing a list of video files to check + * + * @return unprocessed_files A file containing a list of video files that do not have corresponding processed pose files + */ process FILTER_UNPROCESSED_DROPBOX { label "rclone" label "dropbox" @@ -71,6 +98,15 @@ process FILTER_UNPROCESSED_DROPBOX { """ } +/** + * Transfers files between 2 Globus endpoints. + * + * @param globus_src_endpoint The source Globus endpoint to transfer files from + * @param globus_dst_endpoint The destination Globus endpoint to transfer files to + * @param files_to_transfer A file containing a list of files to transfer + * + * @return globus_folder A file containing the path to this tasks folder. + */ process TRANSFER_GLOBUS { label "globus" @@ -115,6 +151,14 @@ process TRANSFER_GLOBUS { """ } +/** + * Retrieves files from Dropbox using rclone. + * + * @param files_to_transfer A file containing a list of files to transfer + * @param dropbox_prefix The rclone remote prefix where files are stored + * + * @return remote_files A file containing a list of the retrieved files with full paths. + */ process GET_DATA_FROM_DROPBOX { label "rclone" label "dropbox" @@ -134,6 +178,15 @@ process GET_DATA_FROM_DROPBOX { """ } +/** + * Uploads a file to Dropbox using rclone. + * + * @param file_to_upload The file to be uploaded + * @param tuple + * - result_file The path to the result file + * - publish_filename The desired publish filename + * @param dropbox_prefix The rclone remote prefix where files are to be uploaded + */ process PUT_DATA_TO_DROPBOX { label "rclone" label "dropbox" diff --git a/nextflow/modules/single_mouse.nf b/nextflow/modules/single_mouse.nf index ddc3940f..181a1c3f 100644 --- a/nextflow/modules/single_mouse.nf +++ b/nextflow/modules/single_mouse.nf @@ -1,3 +1,18 @@ +/** + * This module contains process definitions for single mouse pose estimation. + */ + +/** + * Predicts single mouse segmentation using a pre-trained model. + * + * @param tuple + * - video_file The input video file + * - in_pose_file The input pose file + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with single mouse segmentation predicted. + */ process PREDICT_SINGLE_MOUSE_SEGMENTATION { label "gpu" label "tracking" @@ -16,6 +31,17 @@ process PREDICT_SINGLE_MOUSE_SEGMENTATION { """ } +/** + * Predicts single mouse keypoints using a pre-trained model. + * + * @param tuple + * - video_file The input video file + * - in_pose_file The input pose file + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with single mouse keypoints predicted. + */ process PREDICT_SINGLE_MOUSE_KEYPOINTS { label "gpu" label "tracking" @@ -34,6 +60,15 @@ process PREDICT_SINGLE_MOUSE_KEYPOINTS { """ } +/** + * Performs quality control on single mouse pose files. + * + * @param in_pose_file The input pose file to be checked + * @param clip_duration The duration of the video clip in seconds + * @param batch_name The name of the batch being processed + * + * @return qc_file The generated quality control CSV file + */ process QC_SINGLE_MOUSE { label "tracking" label "r_single_qc" @@ -55,6 +90,18 @@ process QC_SINGLE_MOUSE { """ } +/** + * Clips a video and its corresponding pose file to a specified duration from the start. + * + * @param tuple + * - in_video The input video file to be clipped + * - in_pose_file The input pose file to be clipped + * @param clip_duration The duration in frames to which the video and pose file should be clipped + * + * @return tuple files + * - Path to the trimmed video file. + * - Path to the trimmed pose file. + */ process CLIP_VIDEO_AND_POSE { label "tracking" label "r_clip_video" diff --git a/nextflow/modules/static_objects.nf b/nextflow/modules/static_objects.nf index 21d30323..fce9700d 100644 --- a/nextflow/modules/static_objects.nf +++ b/nextflow/modules/static_objects.nf @@ -1,3 +1,18 @@ +/** + * This module contains process definitions for predicting static objects. + */ + +/** + * Predicts arena corners. + * + * @param tuple + * - video_file Path to the video file. + * - in_pose Path to the input pose file. + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with arena corners predicted. + */ process PREDICT_ARENA_CORNERS { label "gpu" label "tracking" @@ -16,6 +31,17 @@ process PREDICT_ARENA_CORNERS { """ } +/* + * Predicts food hopper position. + * + * @param tuple + * - video_file Path to the video file. + * - in_pose Path to the input pose file. + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with food hopper predicted. + */ process PREDICT_FOOD_HOPPER { label "gpu" label "tracking" @@ -34,6 +60,17 @@ process PREDICT_FOOD_HOPPER { """ } +/** + * Predicts lixit position. + * + * @param tuple + * - video_file Path to the video file. + * - in_pose Path to the input pose file. + * + * @return tuple files + * - Path to the original video file. + * - Modified pose file with lixit predicted. + */ process PREDICT_LIXIT { label "gpu" label "tracking" diff --git a/nextflow/modules/utils.nf b/nextflow/modules/utils.nf index 927a96d3..4ae0fc8e 100644 --- a/nextflow/modules/utils.nf +++ b/nextflow/modules/utils.nf @@ -1,9 +1,15 @@ +/** + * This module contains utility processes and functions for the Nextflow pipeline. + * It includes processes for file manipulation, data validation, and other utility functions. + */ + /** * Lazy nextflow module for creating files, useful for testing. * * @param file_name The name of the file to be created * @param file_content The content to be written to the file - * @return A path to the created file + * + * @return created_file A path to the created file */ process CREATE_FILE { label "r_util" @@ -22,6 +28,15 @@ process CREATE_FILE { """ } +/** + * Filters a batch of files based on whether they exist, and optionally filters out already processed files. + * + * @param input_batch File containing a list of input files to be processed + * @param ignore_invalid_inputs If "true", process will not fail if a file does not exist. Will otherwise remove files that do not exist. + * @param filter_processed If "true", will filter out files that have already been processed + * + * @return process_filelist A file containing a filtered list of files to be processed + */ process FILTER_LOCAL_BATCH { label "r_util" @@ -38,9 +53,13 @@ process FILTER_LOCAL_BATCH { """ touch files_to_process.txt while IFS="" read -r file; do - if [[ ! -f "\${file}" && ${ignore_invalid_inputs} != "true" ]]; then - echo "File does not exist: \${file}" - exit 1 + if [[ ! -f "\${file}" ]]; then + if [[ ${ignore_invalid_inputs} != "true" ]]; then + echo "File does not exist: \${file}" + exit 1 + else + echo "File does not exist: \${file}, skipping." + fi else echo "\${file} exists, adding to process list." echo "\${file}" >> files_to_process.txt @@ -63,6 +82,15 @@ process FILTER_LOCAL_BATCH { """ } +/** + * Generates a dummy pose file for testing purposes. + * + * @param video_file The video file to create a pose file + * + * @return tuple files + * - video_file input video file + * - pose_file pose file + */ process VIDEO_TO_POSE { label "r_util" @@ -80,6 +108,14 @@ process VIDEO_TO_POSE { """ } +/** + * URLifies a file path to avoid potential file collisions in the pipeline. + * + * @param file_to_urlify The path to the file that needs to be URLified + * @param depth The number of directory levels to include in the URLified path + * + * @return file A path to the URLified file + */ process URLIFY_FILE { label "r_util" @@ -98,6 +134,13 @@ process URLIFY_FILE { """ } +/** + * Removes URLified fields from a file path. + * + * @param urlified_file The path to the URLified file that needs to be processed + * + * @return file A path to the file with URLified fields removed + */ process REMOVE_URLIFY_FIELDS { label "r_util" @@ -114,6 +157,15 @@ process REMOVE_URLIFY_FIELDS { """ } +/** + * Merges multiple feature files into a single CSV file by rows. + * + * @param feature_files A list of feature files to be merged + * @param out_filename The name of the output file + * @param header_size The number of header lines to keep from the first file + * + * @return merged_features The merged CSV file + */ process MERGE_FEATURE_ROWS { label "r_util" @@ -137,6 +189,15 @@ process MERGE_FEATURE_ROWS { """ } +/** + * Merges multiple feature files into a single CSV file by columns. + * + * @param feature_files A list of feature files to be merged + * @param col_to_merge_on The column name to merge the files on + * @param out_filename The name of the output file + * + * @return merged_features The merged CSV file + */ process MERGE_FEATURE_COLS { // Any environment with pandas installed should work here. label "tracking" @@ -164,6 +225,15 @@ process MERGE_FEATURE_COLS { """ } +/** + * Selects specific columns from a CSV file and outputs them to a new CSV file. + * + * @param qc_file The input CSV file + * @param key_1 The first column to select + * @param key_2 The second column to select + * + * @return csv_file A CSV file containing only the selected columns + */ process SELECT_COLUMNS { label "r_util" @@ -191,6 +261,15 @@ process SELECT_COLUMNS { """ } +/** + * Adds a new column to a CSV file with specified data. Typically used to add metadata columns. + * + * @param file_to_add_to The CSV file to which the column will be added + * @param column_name The name of the new column + * @param column_data The data to be added in the new column for all rows + * + * @return file A CSV file with the new column added + */ process ADD_COLUMN { label "r_util" @@ -209,6 +288,14 @@ process ADD_COLUMN { """ } +/** + * Deletes a specified row from a CSV file. + * + * @param file_to_delete_from The CSV file from which the row will be deleted + * @param row_to_delete The content of the row to be deleted (exact match) + * + * @return file A CSV file with the specified row removed + */ process DELETE_ROW { label "r_util" @@ -226,6 +313,14 @@ process DELETE_ROW { """ } +/** + * Converts a wide-format feature CSV file to a long-format CSV file. + * + * @param feature_file The input wide-format CSV file + * @param id_col The column to use as the identifier in the long format + * + * @return long_file A long-format CSV file + */ process FEATURE_TO_LONG { // Any environment with pandas installed should work here. label "tracking" @@ -249,6 +344,16 @@ process FEATURE_TO_LONG { """ } +/** + * Converts a long-format feature CSV file to a wide-format CSV file. + * + * @param long_file The input long-format CSV file + * @param id_col The column to use as the identifier in the wide format + * @param feature_col The column containing feature names + * @param value_col The column containing feature values + * + * @return wide_file A wide-format CSV file + */ process LONG_TO_WIDE { // Any environment with pandas installed should work here. label "tracking" @@ -274,28 +379,17 @@ process LONG_TO_WIDE { """ } -process SUBSET_PATH_BY_VAR { - label "r_util" - - input: - path all_files - val subset_files - val dir - - output: - path("${dir}/*"), emit: subset_files - - script: - """ - mkdir ${dir} - for file in ${subset_files} - do - ln -s \$(pwd)/\${file} ${dir}/\$(basename \${file}) - done - sleep 10 - """ -} - +/** + * Publishes a result file to a specified directory with potential sub-folders. + * + * @param tuple + * - result_file The file to be published + * - publish_filename The name under which the file will be published + * + * @return published_file A path to the published file + * + * @publish ./ File published to the specified directory + */ process PUBLISH_RESULT_FILE { label "r_util" @@ -319,6 +413,14 @@ process PUBLISH_RESULT_FILE { """ } +/** + * Obtains workflow version information and writes it to a file. + * + * @return version The workflow version + * @return version_file Path to the version file + * + * @publish ./ Workflow version information + */ process GET_WORKFLOW_VERSION { label "r_util" @@ -338,6 +440,16 @@ process GET_WORKFLOW_VERSION { """ } +/** + * Pairs a pose file with a generated video file. + * + * @param pose_file The pose file for which to create a video + * @param n_frames The number of frames the video should have + * + * @return tuple files + * - video_file The created video file + * - pose_file The input pose file + */ process ADD_DUMMY_VIDEO { // Any environment with ffmpeg installed should work here. label "tracking" @@ -361,6 +473,7 @@ process ADD_DUMMY_VIDEO { * * @param file_path The path to the file that needs validation * @param pipeline_type The type of pipeline being run (e.g. 'single-mouse', 'single-mouse-corrected-corners', etc.) + * * @return A boolean indicating if the file is valid and an error message if it's not */ def validateInputFile(String file_path, String pipeline_type) { @@ -392,6 +505,7 @@ def validateInputFile(String file_path, String pipeline_type) { * * @param in_file_list The path to the file that contains the list of intut files * @param pipeline_type The type of pipeline being run. See validateInputFile for valid types. + * * @return A list of valid file paths that match the allowed formats for the specified pipeline type. */ def validateInputFilelist(String in_file_list, String pipeline_type) { diff --git a/nextflow/workflows/feature_generation.nf b/nextflow/workflows/feature_generation.nf index 1a4d055c..f3651273 100644 --- a/nextflow/workflows/feature_generation.nf +++ b/nextflow/workflows/feature_generation.nf @@ -1,3 +1,6 @@ +/** + * This module contains workflows related to feature generation from pose files. + */ include { GENERATE_GAIT_H5; GENERATE_GAIT_BIN } from "${projectDir}/nextflow/modules/gait" include { GENERATE_FLEXIBILITY_INDEX; GENERATE_REAR_PAW_WIDTH } from "${projectDir}/nextflow/modules/flexibility" include { GET_WORKFLOW_VERSION; @@ -32,6 +35,20 @@ include { GENERATE_FEATURE_CACHE; BEHAVIOR_TABLE_TO_FEATURES } from "${projectDir}/nextflow/modules/jabs_classifiers" include { EXTRACT_FECAL_BOLI_BINS } from "${projectDir}/nextflow/modules/fecal_boli" +/** + * Workflow to generate features from single mouse pose v2 files. + * Generates gait and morphometric feature sets. + * + * @param tuple input_pose_v2_batch + * - Path to the input video file. + * - Path to the corresponding pose v2 file. + * + * @return path gait_results The channel of generated gait feature files. + * @return path morphometrics_results The channel of generated morphometric feature files. + * + * @publish ./results/ Gait feature files + * @publish ./results/ Morphometric feature files + */ workflow SINGLE_MOUSE_V2_FEATURES { take: // tuple of video_file and pose_file from SINGLE_MOUSE_TRACKING @@ -74,6 +91,20 @@ workflow SINGLE_MOUSE_V2_FEATURES { morphometrics_results } +/** + * Workflow to generate JABS features from single mouse pose v6 files. + * Generates heuristic and classifier-based behavior features, as well as fecal boli counts. + * + * @param tuple input_pose_v6_batch + * - Path to the input video file. + * - Path to the corresponding pose v6 file. + * + * @return path wide_jabs_features The channel of generated JABS feature files. + * @return path fecal_boli_table The channel of generated fecal boli count files. + * + * @publish ./results/ JABS feature files + * @publish ./results/ Fecal boli count files + */ workflow SINGLE_MOUSE_V6_FEATURES { take: // tuple of video_file and pose_file from SINGLE_MOUSE_TRACKING diff --git a/nextflow/workflows/io.nf b/nextflow/workflows/io.nf index 693fb2be..a121c966 100644 --- a/nextflow/workflows/io.nf +++ b/nextflow/workflows/io.nf @@ -1,3 +1,7 @@ +/** + * This module contains workflows that manipulate inputs and outputs to pipelines. + */ + include { FILTER_LOCAL_BATCH; URLIFY_FILE; validateInputFile; @@ -9,6 +13,19 @@ include { CHECK_GLOBUS_AUTH; GET_DATA_FROM_DROPBOX; } from "${projectDir}/nextflow/modules/remote_io" +/** + * Prepares data for the main pipeline. Steps include: + * - Validating input files by extension + * - [Optional] Filtering of already processed files + * - Retrieval of files if remote + * - URLifying file names to avoid collisions + * + * @param path in_video_file Text file containing list of files to process. One file per line. + * @param val location The location of the input files (local, dropbox, or globus). + * @param val skip_urlify If true, skips the URLification step. Some usages may already have URLified files. + * + * @return path file_processing_channel The channel containing the prepared input files. + */ workflow PREPARE_DATA { take: in_video_file diff --git a/nextflow/workflows/multi_mouse_pipeline.nf b/nextflow/workflows/multi_mouse_pipeline.nf index 9c71d7f3..8ed447f6 100644 --- a/nextflow/workflows/multi_mouse_pipeline.nf +++ b/nextflow/workflows/multi_mouse_pipeline.nf @@ -1,3 +1,7 @@ +/** + * This module contains the multi-mouse tracking pipeline. + * It processes video input to track multiple mice, predict their poses, and identify static objects. + */ include { PREDICT_MULTI_MOUSE_SEGMENTATION; PREDICT_MULTI_MOUSE_KEYPOINTS; PREDICT_MULTI_MOUSE_IDENTITY; @@ -11,6 +15,18 @@ include { VIDEO_TO_POSE; PUBLISH_RESULT_FILE as PUBLISH_MM_POSE_V6; } from "${projectDir}/nextflow/modules/utils" +/** + * Main workflow for predicting multimouse poses from video input. + * + * @param path input_video The input video file to process. + * @param val num_animals The number of animals to track in the video. + * + * @return tuple pose_v6 + * - Path to the input video file. + * - Path to the pose v6 file produced. + * + * @publish ./results/ Pose v6 results + */ workflow MULTI_MOUSE_TRACKING { take: input_video diff --git a/nextflow/workflows/single_mouse_pipeline.nf b/nextflow/workflows/single_mouse_pipeline.nf index 676762b2..26ce03f1 100644 --- a/nextflow/workflows/single_mouse_pipeline.nf +++ b/nextflow/workflows/single_mouse_pipeline.nf @@ -1,3 +1,7 @@ +/** + * This module contains the single mouse tracking pipeline. + * It processes video input to track a single mouse. + */ include { PREDICT_SINGLE_MOUSE_SEGMENTATION; PREDICT_SINGLE_MOUSE_KEYPOINTS; CLIP_VIDEO_AND_POSE } from "${projectDir}/nextflow/modules/single_mouse" include { PREDICT_ARENA_CORNERS } from "${projectDir}/nextflow/modules/static_objects" include { PREDICT_FECAL_BOLI } from "${projectDir}/nextflow/modules/fecal_boli" @@ -14,6 +18,21 @@ include { VIDEO_TO_POSE; PUBLISH_RESULT_FILE as PUBLISH_SM_POSE_V6_NOCORN; } from "${projectDir}/nextflow/modules/utils" +/** + * Main workflow for single mouse tracking. + * + * @param path input_video The input video file to process. + * + * @return tuple pose_v2_data + * - Path to the input video file. + * - Path to the pose v2 file produced. + * @return tuple pose_v6_data + * - Path to the input video file. + * - Path to the pose v6 file produced. + * + * @publish ./results/ Trimmed video files + * @publish ./results/ Single mouse pose v2 results + */ workflow SINGLE_MOUSE_TRACKING { take: input_video @@ -46,6 +65,26 @@ workflow SINGLE_MOUSE_TRACKING { pose_v6_data } +/** + * Workflow to split pose files based on whether corners were detected. + * If corners are missing, the pose file is sent for manual correction. + * Default files are provided if either channel in the split is empty. + * + * @param tuple input_pose_v6_batch + * - Path to the input video file. + * - Path to the pose v6 file produced. + * + * @return tuple v6_with_corners + * - Path to the input video file. + * - Path to the pose v6 file with corners present. + * @return tuple v6_without_corners + * - Path to the input video file. + * - Path to the pose v6 file without corners present. + * + * @publish ./ Single mouse quality control results + * @publish ./results/ Single mouse pose v6 results with corners + * @publish ./failed_corners/ Single mouse pose v6 results without corners. Files are url-ified. + */ workflow SPLIT_BY_CORNERS { take: input_pose_v6_batch diff --git a/nextflow/workflows/sleap_manual_correction.nf b/nextflow/workflows/sleap_manual_correction.nf index 8560a4e3..6654a8e7 100644 --- a/nextflow/workflows/sleap_manual_correction.nf +++ b/nextflow/workflows/sleap_manual_correction.nf @@ -1,6 +1,19 @@ +/** + * This module contains workflows for manual correction of arena corner annotations. + */ include { EXTRACT_VIDEO_FRAME; ADD_EXAMPLES_TO_SLEAP; INTEGRATE_SLEAP_CORNER_ANNOTATIONS } from "${projectDir}/nextflow/modules/manual_correction" include { PUBLISH_RESULT_FILE as PUBLISH_SM_MANUAL_CORRECT } from "${projectDir}/nextflow/modules/utils" +/** + * Workflow to generate a SLEAP file for manual correction of arena corners. + * + * @param path input_files Channel of input video files to extract frames from. + * @param val frame_index The index of the frame to extract from each video. + * + * @return path sleap_file The SLEAP file containing the extracted frames for manual correction. + * + * @publish ./ Manual corner correction SLEAP file + */ workflow MANUALLY_CORRECT_CORNERS { take: input_files @@ -18,6 +31,14 @@ workflow MANUALLY_CORRECT_CORNERS { sleap_file } +/** + * Workflow to integrate manually corrected corner annotations from a SLEAP file back into pose files. + * + * @param path pose_files Channel of pose files to update with corrected corner annotations. + * @param path sleap_file The SLEAP file containing the manually corrected corner annotations. + * + * @return path corrected_poses The channel of pose files updated with the corrected corner annotations. + */ workflow INTEGRATE_CORNER_ANNOTATIONS { take: pose_files