diff --git a/.circleci/ds005_fasttrack_outputs.txt b/.circleci/ds005_fasttrack_outputs.txt index d7ca30b31..8903c6ebe 100644 --- a/.circleci/ds005_fasttrack_outputs.txt +++ b/.circleci/ds005_fasttrack_outputs.txt @@ -3,6 +3,7 @@ dataset_description.json desc-aparcaseg_dseg.tsv desc-aseg_dseg.tsv +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/.circleci/ds005_outputs.txt b/.circleci/ds005_outputs.txt index 81ceb28ac..308f19adb 100644 --- a/.circleci/ds005_outputs.txt +++ b/.circleci/ds005_outputs.txt @@ -3,6 +3,7 @@ dataset_description.json desc-aparcaseg_dseg.tsv desc-aseg_dseg.tsv +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/.circleci/ds005_partial_fasttrack_outputs.txt b/.circleci/ds005_partial_fasttrack_outputs.txt index 899352a35..554148b11 100644 --- a/.circleci/ds005_partial_fasttrack_outputs.txt +++ b/.circleci/ds005_partial_fasttrack_outputs.txt @@ -3,6 +3,7 @@ dataset_description.json desc-aparcaseg_dseg.tsv desc-aseg_dseg.tsv +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/.circleci/ds005_partial_outputs.txt b/.circleci/ds005_partial_outputs.txt index ba934e6fd..35576a5ae 100644 --- a/.circleci/ds005_partial_outputs.txt +++ b/.circleci/ds005_partial_outputs.txt @@ -3,6 +3,7 @@ dataset_description.json desc-aparcaseg_dseg.tsv desc-aseg_dseg.tsv +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/.circleci/ds054_fasttrack_outputs.txt b/.circleci/ds054_fasttrack_outputs.txt index c806b68e9..5c378ff1d 100644 --- a/.circleci/ds054_fasttrack_outputs.txt +++ b/.circleci/ds054_fasttrack_outputs.txt @@ -1,6 +1,7 @@ .bidsignore dataset_description.json +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/.circleci/ds054_outputs.txt b/.circleci/ds054_outputs.txt index f50701f6f..ed7e3d060 100644 --- a/.circleci/ds054_outputs.txt +++ b/.circleci/ds054_outputs.txt @@ -1,6 +1,7 @@ .bidsignore dataset_description.json +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/.circleci/ds210_fasttrack_outputs.txt b/.circleci/ds210_fasttrack_outputs.txt index e5361b1f8..63c481668 100644 --- a/.circleci/ds210_fasttrack_outputs.txt +++ b/.circleci/ds210_fasttrack_outputs.txt @@ -1,6 +1,7 @@ .bidsignore dataset_description.json +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/.circleci/ds210_outputs.txt b/.circleci/ds210_outputs.txt index b0439969b..4353e5e92 100644 --- a/.circleci/ds210_outputs.txt +++ b/.circleci/ds210_outputs.txt @@ -1,6 +1,7 @@ .bidsignore dataset_description.json +descriptions.tsv logs logs/CITATION.bib logs/CITATION.html diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 27a2c22b3..cfbcf6472 100644 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -33,7 +33,11 @@ def main(): from os import EX_SOFTWARE from pathlib import Path - from ..utils.bids import write_bidsignore, write_derivative_description + from ..utils.bids import ( + write_bidsignore, + write_derivative_description, + write_descriptions_tsv, + ) from .parser import parse_args from .workflow import build_workflow @@ -225,6 +229,18 @@ def main(): ) write_bidsignore(config.execution.fmriprep_dir) + # Write descriptions.tsv documenting desc- entity procedures + write_descriptions_tsv( + config.execution.fmriprep_dir, + slice_timing_corrected='slicetiming' not in (config.workflow.ignore or []), + slice_time_ref=config.workflow.slice_time_ref, + fd_threshold=config.workflow.regressors_fd_th, + dvars_threshold=config.workflow.regressors_dvars_th, + coreg_method='bbr' if config.workflow.use_bbr else 'header', + bold2anat_dof=config.workflow.bold2anat_dof or 6, + freesurfer=config.workflow.run_reconall, + ) + if failed_reports: msg = ( 'Report generation was not successful for the following participants ' diff --git a/fmriprep/data/descriptions.json b/fmriprep/data/descriptions.json new file mode 100644 index 000000000..c9fd934b9 --- /dev/null +++ b/fmriprep/data/descriptions.json @@ -0,0 +1,101 @@ +{ + "_version": "1.0", + "_note": "Procedure-focused descriptions for BIDS desc- entities. Descriptions explain WHAT PROCESSING was done, not what the data represents. The suffix (e.g., _bold, _mask) indicates what the data IS; the desc- indicates what procedure created it.", + + "entities": { + "preproc": { + "base": "Preprocessing pipeline applied:", + "conditional_parts": [ + {"condition": "always", "text": " head motion correction"}, + {"condition": "sdc_applied", "text": ", susceptibility distortion correction ({sdc_method})"}, + {"condition": "stc_applied", "text": ", slice timing correction (reference: {slice_time_ref})"} + ], + "parameters": ["sdc_method", "slice_time_ref"] + }, + "brain": { + "base": "Brain extraction via {mask_source} segmentation, projected and binarized", + "parameters": ["mask_source"] + }, + "hmc": { + "base": "Rigid-body volume-to-reference registration for head motion estimation", + "parameters": [] + }, + "coreg": { + "base": "Functional-to-anatomical alignment via {coreg_method}", + "conditional_parts": [ + {"condition": "has_dof", "text": " ({dof} degrees of freedom)"} + ], + "parameters": ["coreg_method", "dof"] + }, + "fmap": { + "base": "Fieldmap estimation and registration to functional reference", + "parameters": [] + }, + "confounds": { + "base": "Nuisance regressor extraction:", + "conditional_parts": [ + {"condition": "always", "text": " head motion parameters"}, + {"condition": "has_compcor", "text": ", CompCor components (variance threshold: {compcor_variance})"}, + {"condition": "always", "text": ", tissue-averaged signals"}, + {"condition": "has_fd_threshold", "text": "; outlier flagging at FD > {fd_threshold} mm"}, + {"condition": "has_dvars_threshold", "text": ", DVARS > {dvars_threshold}"} + ], + "parameters": ["fd_threshold", "dvars_threshold", "compcor_variance"] + }, + "goodvoxels": { + "base": "Voxel exclusion based on local coefficient of variation threshold (HCP-style surface projection)", + "parameters": [] + }, + "summary": { + "base": "Aggregation of processing parameters and metadata for quality review", + "parameters": [] + }, + "validation": { + "base": "Automated input data validation and metadata consistency checks", + "parameters": [] + }, + "fmapCoreg": { + "base": "Visualization of fieldmap-to-functional reference alignment quality", + "parameters": [] + }, + "sdc": { + "base": "Before/after comparison of susceptibility distortion correction", + "conditional_parts": [ + {"condition": "sdc_applied", "text": " ({sdc_method})"} + ], + "parameters": ["sdc_method"] + }, + "rois": { + "base": "Visualization of anatomical ROI masks used for signal extraction", + "parameters": [] + }, + "compcorvar": { + "base": "Cumulative variance plot for CompCor component selection criterion", + "parameters": [] + }, + "confoundcorr": { + "base": "Pairwise correlation analysis of extracted nuisance timeseries", + "parameters": [] + }, + "carpetplot": { + "base": "Grayplot (carpet plot) generation with motion and signal quality traces", + "parameters": [] + }, + "t2scomp": { + "base": "Multi-echo T2* estimation quality visualization via echo comparison", + "parameters": [] + }, + "t2starhist": { + "base": "Histogram analysis of estimated T2* values across brain tissue", + "parameters": [] + }, + "aparcaseg": { + "base": "FreeSurfer cortical parcellation projection to functional space", + "parameters": [] + }, + "aseg": { + "base": "FreeSurfer subcortical segmentation projection to functional space", + "parameters": [] + } + } +} diff --git a/fmriprep/tests/test_descriptions.py b/fmriprep/tests/test_descriptions.py new file mode 100644 index 000000000..ebb64f7bb --- /dev/null +++ b/fmriprep/tests/test_descriptions.py @@ -0,0 +1,252 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +# +# Copyright The NiPreps Developers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for descriptions.tsv generation.""" + +from __future__ import annotations + +import json + +import pytest + +from fmriprep.data import load as load_data +from fmriprep.utils.bids import write_descriptions_tsv + + +def test_descriptions_json_loads(): + """Test that descriptions.json loads correctly.""" + desc_data = json.loads(load_data.readable('descriptions.json').read_text()) + + assert '_version' in desc_data + assert 'entities' in desc_data + assert len(desc_data['entities']) > 0 + + # Check that required entities exist + required_entities = ['preproc', 'brain', 'hmc', 'coreg', 'confounds'] + for entity in required_entities: + assert entity in desc_data['entities'], f'Missing required entity: {entity}' + + +def test_descriptions_json_structure(): + """Test that each entity in descriptions.json has correct structure.""" + desc_data = json.loads(load_data.readable('descriptions.json').read_text()) + + for entity_id, entity_def in desc_data['entities'].items(): + # Each entity must have a 'base' description + assert 'base' in entity_def, f"Entity '{entity_id}' missing 'base' field" + assert isinstance(entity_def['base'], str), f"Entity '{entity_id}' base must be string" + + # Parameters must be a list if present + if 'parameters' in entity_def: + assert isinstance(entity_def['parameters'], list), ( + f"Entity '{entity_id}' parameters must be list" + ) + + # Conditional parts must have correct structure if present + if 'conditional_parts' in entity_def: + assert isinstance(entity_def['conditional_parts'], list), ( + f"Entity '{entity_id}' conditional_parts must be list" + ) + for part in entity_def['conditional_parts']: + assert 'condition' in part, f"Entity '{entity_id}' part missing condition" + assert 'text' in part, f"Entity '{entity_id}' part missing text" + + +def test_write_descriptions_tsv_basic(tmp_path): + """Test basic descriptions.tsv generation.""" + tsv_path = write_descriptions_tsv(tmp_path) + + assert tsv_path.exists() + assert tsv_path.name == 'descriptions.tsv' + + # Read and parse TSV + content = tsv_path.read_text() + lines = content.strip().split('\n') + + # Check header + assert lines[0] == 'desc_id\tdescription\tparameters' + + # Check that we have data rows + assert len(lines) > 1 + + # Parse rows + for line in lines[1:]: + parts = line.split('\t') + assert len(parts) == 3, f'Each row should have 3 columns: {line}' + _desc_id, _description, parameters = parts + + # Check parameters is valid JSON + json.loads(parameters) + + +def test_write_descriptions_tsv_with_sdc(tmp_path): + """Test descriptions.tsv with SDC enabled.""" + tsv_path = write_descriptions_tsv( + tmp_path, + sdc_method='TOPUP', + slice_timing_corrected=True, + slice_time_ref=0.5, + ) + + content = tsv_path.read_text() + + # Find preproc row + for line in content.split('\n'): + if line.startswith('preproc\t'): + parts = line.split('\t') + description = parts[1] + parameters = json.loads(parts[2]) + + # SDC should be mentioned in description + assert 'susceptibility distortion correction' in description.lower() + assert 'TOPUP' in description + + # Slice timing should be mentioned + assert 'slice timing correction' in description.lower() + + # Check parameters + assert parameters['sdc_applied'] is True + assert parameters['stc_applied'] is True + assert parameters['sdc_method'] == 'TOPUP' + break + else: + pytest.fail('preproc row not found') + + +def test_write_descriptions_tsv_without_sdc(tmp_path): + """Test descriptions.tsv without SDC (SDC not applied).""" + tsv_path = write_descriptions_tsv( + tmp_path, + sdc_method=None, # No SDC + slice_timing_corrected=False, + ) + + content = tsv_path.read_text() + + # Find preproc row + for line in content.split('\n'): + if line.startswith('preproc\t'): + parts = line.split('\t') + description = parts[1] + parameters = json.loads(parts[2]) + + # SDC should NOT be mentioned in description + assert 'susceptibility distortion correction' not in description.lower() + + # Slice timing should NOT be mentioned + assert 'slice timing correction' not in description.lower() + + # Check parameters + assert parameters['sdc_applied'] is False + assert parameters['stc_applied'] is False + break + else: + pytest.fail('preproc row not found') + + +def test_write_descriptions_tsv_freesurfer_mask(tmp_path): + """Test that FreeSurfer mask source is reflected in description.""" + tsv_path = write_descriptions_tsv(tmp_path, freesurfer=True) + + content = tsv_path.read_text() + + for line in content.split('\n'): + if line.startswith('brain\t'): + description = line.split('\t')[1] + assert 'FreeSurfer' in description + break + + +def test_write_descriptions_tsv_ants_mask(tmp_path): + """Test that ANTs mask source is reflected in description.""" + tsv_path = write_descriptions_tsv(tmp_path, freesurfer=False) + + content = tsv_path.read_text() + + for line in content.split('\n'): + if line.startswith('brain\t'): + description = line.split('\t')[1] + assert 'ANTs' in description + break + + +def test_write_descriptions_tsv_confounds(tmp_path): + """Test confounds description includes thresholds when provided.""" + tsv_path = write_descriptions_tsv( + tmp_path, + fd_threshold=0.5, + dvars_threshold=1.5, + compcor_variance=0.5, + ) + + content = tsv_path.read_text() + + for line in content.split('\n'): + if line.startswith('confounds\t'): + parts = line.split('\t') + description = parts[1] + parameters = json.loads(parts[2]) + + # Check description mentions thresholds + assert 'FD' in description + assert '0.5' in description + assert 'DVARS' in description + assert '1.5' in description + + # Check parameters + assert parameters['fd_threshold'] == 0.5 + assert parameters['dvars_threshold'] == 1.5 + break + + +def test_write_descriptions_tsv_coreg_bbr(tmp_path): + """Test coregistration description with BBR.""" + tsv_path = write_descriptions_tsv( + tmp_path, + coreg_method='bbr', + bold2anat_dof=6, + ) + + content = tsv_path.read_text() + + for line in content.split('\n'): + if line.startswith('coreg\t'): + description = line.split('\t')[1] + assert 'boundary-based registration' in description + assert '6' in description + assert 'degrees of freedom' in description + break + + +def test_write_descriptions_tsv_all_entities_present(tmp_path): + """Test that all defined entities are present in output.""" + desc_data = json.loads(load_data.readable('descriptions.json').read_text()) + expected_entities = set(desc_data['entities'].keys()) + + tsv_path = write_descriptions_tsv(tmp_path) + content = tsv_path.read_text() + + found_entities = set() + for line in content.split('\n')[1:]: # Skip header + if line.strip(): + desc_id = line.split('\t')[0] + found_entities.add(desc_id) + + assert found_entities == expected_entities, ( + f'Missing entities: {expected_entities - found_entities}, ' + f'Extra entities: {found_entities - expected_entities}' + ) diff --git a/fmriprep/utils/bids.py b/fmriprep/utils/bids.py index 708f283c5..d2653a4e8 100644 --- a/fmriprep/utils/bids.py +++ b/fmriprep/utils/bids.py @@ -139,6 +139,143 @@ def write_bidsignore(deriv_dir): ignore_file.write_text('\n'.join(bids_ignore) + '\n') +def write_descriptions_tsv( + deriv_dir, + *, + sdc_method: str | None = None, + slice_timing_corrected: bool = False, + slice_time_ref: float = 0.5, + fd_threshold: float | None = None, + dvars_threshold: float | None = None, + compcor_variance: float = 0.5, + coreg_method: str = 'bbr', + bold2anat_dof: int = 6, + freesurfer: bool = True, +): + """Write descriptions.tsv to derivatives root directory. + + This file documents the processing procedures associated with each + ``desc-`` entity value used in the derivatives, following the BIDS + common derivatives specification. + + Parameters + ---------- + deriv_dir : :obj:`str` or :obj:`os.PathLike` + Root directory of fMRIPrep derivatives + sdc_method : :obj:`str`, optional + Susceptibility distortion correction method used (e.g., 'TOPUP', 'SyN'). + If None, SDC was not applied. + slice_timing_corrected : :obj:`bool` + Whether slice timing correction was applied + slice_time_ref : :obj:`float` + Slice timing reference (0-1), only used if slice_timing_corrected is True + fd_threshold : :obj:`float`, optional + Framewise displacement threshold in mm for outlier flagging + dvars_threshold : :obj:`float`, optional + DVARS threshold for outlier detection + compcor_variance : :obj:`float` + Variance threshold for CompCor component selection (default 0.5 = 50%) + coreg_method : :obj:`str` + Coregistration method (e.g., 'bbr' for boundary-based registration) + bold2anat_dof : :obj:`int` + Degrees of freedom for BOLD-to-anatomical registration + freesurfer : :obj:`bool` + Whether FreeSurfer was used for segmentation + + Returns + ------- + :obj:`pathlib.Path` + Path to written descriptions.tsv + + """ + deriv_dir = Path(deriv_dir) + + # Load description templates + desc_templates = json.loads(load_data.readable('descriptions.json').read_text()) + entities = desc_templates.get('entities', {}) + + # Build context for template substitution + context = { + 'sdc_applied': sdc_method is not None, + 'sdc_method': sdc_method or '', + 'stc_applied': slice_timing_corrected, + 'slice_time_ref': slice_time_ref, + 'has_fd_threshold': fd_threshold is not None, + 'fd_threshold': fd_threshold if fd_threshold is not None else '', + 'has_dvars_threshold': dvars_threshold is not None, + 'dvars_threshold': dvars_threshold if dvars_threshold is not None else '', + 'has_compcor': True, # CompCor is always computed + 'compcor_variance': compcor_variance, + 'coreg_method': 'boundary-based registration' if coreg_method == 'bbr' else coreg_method, + 'has_dof': bold2anat_dof is not None, + 'dof': bold2anat_dof, + 'mask_source': 'FreeSurfer' if freesurfer else 'ANTs', + } + + def _build_description(entity_def: dict, ctx: dict) -> str: + """Build description string from template definition.""" + desc = entity_def.get('base', '') + + # Process conditional parts + for part in entity_def.get('conditional_parts', []): + condition = part.get('condition', '') + text = part.get('text', '') + + # Check if condition is met + include = False + if condition == 'always': + include = True + elif condition in ctx: + include = bool(ctx[condition]) + + if include: + # Substitute parameters in text + try: + desc += text.format(**ctx) + except KeyError: + desc += text + + # Substitute any remaining parameters in base + try: + desc = desc.format(**ctx) + except KeyError: + pass + + return desc + + # Build rows for descriptions.tsv + rows = [] + for desc_id, entity_def in entities.items(): + description = _build_description(entity_def, context) + + # Build parameters dict (only include non-empty values) + params = {} + for param_name in entity_def.get('parameters', []): + if param_name in context and context[param_name] not in (None, ''): + params[param_name] = context[param_name] + + # Add condition flags to parameters for key dynamic fields + if desc_id == 'preproc': + params['sdc_applied'] = context['sdc_applied'] + params['stc_applied'] = context['stc_applied'] + + rows.append( + { + 'desc_id': desc_id, + 'description': description, + 'parameters': json.dumps(params) if params else '{}', + } + ) + + # Write TSV file + tsv_path = deriv_dir / 'descriptions.tsv' + header = 'desc_id\tdescription\tparameters\n' + lines = [f'{row["desc_id"]}\t{row["description"]}\t{row["parameters"]}\n' for row in rows] + tsv_path.write_text(header + ''.join(lines)) + + return tsv_path + + def write_derivative_description(bids_dir, deriv_dir, dataset_links=None): from .. import __version__