From bfee32c8040bcad1d7f7b6f46f8aa3f91f15e6e8 Mon Sep 17 00:00:00 2001 From: Nurun Nahar Fancy Date: Fri, 24 Mar 2023 13:44:21 +0000 Subject: [PATCH 01/12] run with v 7.2.0 --- .github/workflows/ci.yml | 6 +++--- conf/test.config | 4 ++-- nextflow.config | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ba6b78..9d0eff7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,9 @@ jobs: - name: Build new docker image if: ${{ github.event.workflow_run.conclusion == 'thiswillnevertrigger' }} - run: docker build --no-cache . -t almurphy/scfdev:dev + run: docker build --no-cache . -t nfancy/scflow:dev - name: Pull docker image if: ${{ github.event.workflow_run.conclusion == 'thiswillnevertrigger' }} run: | - docker pull almurphy/scfdev:dev - docker tag almurphy/scfdev:dev almurphy/scfdev:dev + docker pull nfancy/scflow:dev + docker tag nfancy/scflow:dev nfancy/scflow:dev diff --git a/conf/test.config b/conf/test.config index 945ee35..b31350b 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,8 +20,8 @@ params { input = "https://raw.githubusercontent.com/nf-core/test-datasets/scflow/refs/SampleSheet.tsv" manifest = "https://raw.githubusercontent.com/nf-core/test-datasets/scflow/refs/Manifest.txt" - ensembl_mappings = "https://raw.githubusercontent.com/nfancy/test-datasets/scflow/assets/ensembl_mappings.tsv" - ctd_path = "https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/28033407/ctd_v1.zip" + ensembl_mappings = "https://raw.githubusercontent.com/test-datasets/scflow/assets/ensembl_mappings.tsv" + ctd_path = "https://raw.githubusercontent.com/combiz/scFlowData/dev-nf/assets/ctd.zip" reddim_genes_yml = "https://raw.githubusercontent.com/nf-core/test-datasets/scflow/refs/reddim_genes.yml" reddimplot_pointsize = 1 diff --git a/nextflow.config b/nextflow.config index c90fd42..7a7f01a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -18,7 +18,7 @@ manifest { // Container slug. Stable releases should specify release tag! // Developmental code should specify :dev -process.container = 'almurphy/scfdev:0.7.1' +process.container = 'nfancy/scflow:0.7.2' //workDir = "/rds/general/user/$USER/ephemeral/tmp" workDir = './work' @@ -29,7 +29,7 @@ params { manifest = './refs/Manifest.txt' input = './refs/SampleSheet.tsv' ensembl_mappings = './src/ensembl-ids/ensembl_mappings.tsv' - ctd_path = 'https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/28033407/ctd_v1.zip' + ctd_path = 'https://raw.githubusercontent.com/combiz/scFlowData/dev-nf/assets/ctd.zip' celltype_mappings = './conf/celltype_mappings.tsv' reddim_genes_yml = './conf/reddim_genes.yml' From 67693d7794f0ae8ef89f1b2368016c1f8d3759e8 Mon Sep 17 00:00:00 2001 From: Nurun Nahar Fancy Date: Fri, 24 Mar 2023 13:52:18 +0000 Subject: [PATCH 02/12] run test --- conf/test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/test.config b/conf/test.config index b31350b..8e9a385 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,7 +20,7 @@ params { input = "https://raw.githubusercontent.com/nf-core/test-datasets/scflow/refs/SampleSheet.tsv" manifest = "https://raw.githubusercontent.com/nf-core/test-datasets/scflow/refs/Manifest.txt" - ensembl_mappings = "https://raw.githubusercontent.com/test-datasets/scflow/assets/ensembl_mappings.tsv" + ensembl_mappings = "https://raw.githubusercontent.com/nf-core/test-datasets/scflow/assets/ensembl_mappings.tsv" ctd_path = "https://raw.githubusercontent.com/combiz/scFlowData/dev-nf/assets/ctd.zip" reddim_genes_yml = "https://raw.githubusercontent.com/nf-core/test-datasets/scflow/refs/reddim_genes.yml" From eddcddd120b3fe5365cd673a857d63a5a2dd46cc Mon Sep 17 00:00:00 2001 From: Nurun Nahar Fancy Date: Fri, 24 Mar 2023 14:11:06 +0000 Subject: [PATCH 03/12] fixed param for reduce_dim_sce --- conf/scflow_analysis.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/scflow_analysis.config b/conf/scflow_analysis.config index 3af4bdd..79493ec 100644 --- a/conf/scflow_analysis.config +++ b/conf/scflow_analysis.config @@ -73,7 +73,7 @@ params { // Options: Dimensionality Reduction reddim_input_reduced_dim = 'PCA,Liger' // * reddim_reduction_methods = 'tSNE,UMAP,UMAP3D' // * - reddim_vars_to_regress_out = 'nCount_RNA,pc_mito' // * + reddim_vars_to_regress_out = 'total_counts,pc_mito' // * // umap reddim_umap_pca_dims = 30 reddim_umap_n_neighbors = 35 From 64613113d6d0bc2626ea187871584681df614d81 Mon Sep 17 00:00:00 2001 From: nfancy Date: Thu, 13 Apr 2023 09:51:36 +0000 Subject: [PATCH 04/12] updated with scflow v0.7.2 --- .editorconfig | 0 .gitattributes | 0 .github/.dockstore.yml | 0 .github/CONTRIBUTING.md | 0 .github/ISSUE_TEMPLATE/bug_report.md | 0 .github/ISSUE_TEMPLATE/config.yml | 0 .github/ISSUE_TEMPLATE/feature_request.md | 0 .github/PULL_REQUEST_TEMPLATE.md | 0 .github/workflows/awsfulltest.yml | 0 .github/workflows/awstest.yml | 0 .github/workflows/branch.yml | 0 .github/workflows/ci.yml | 0 .github/workflows/linting.yml | 0 .github/workflows/linting_comment.yml | 0 .gitignore | 0 .markdownlint.yml | 0 .nf-core.yml | 0 CHANGELOG.md | 0 CITATIONS.md | 0 CODE_OF_CONDUCT.md | 0 LICENSE | 0 README.md | 0 assets/NO_FILE.tsv | 0 assets/email_template.html | 0 assets/email_template.txt | 0 assets/nf-core-scflow_logo.png | Bin assets/schema_input.json | 0 assets/sendmail_template.txt | 0 bin/scflow_dge.r | 14 ++++++-------- bin/scflow_ipa.r | 8 ++++---- bin/scflow_map_celltypes.r | 10 ++++++++++ bin/scflow_reduce_dims.r | 1 + conf/base.config | 0 conf/gcp.config | 0 conf/igenomes.config | 0 conf/modules.config | 0 conf/scflow_analysis.config | 5 +++-- conf/test.config | 0 conf/test_full.config | 0 docs/README.md | 0 docs/images/nf-core-scflow_logo.png | Bin docs/images/scflow_workflow.png | Bin docs/output.md | 0 docs/usage.md | 0 lib/nfcore_external_java_deps.jar | Bin main.nf | 0 modules.json | 0 modules/local/functions.nf | 0 modules/local/get_software_versions.nf | 0 modules/local/process/functions.nf | 0 modules/local/process/scflow/checkinputs.nf | 0 modules/local/process/scflow/cluster.nf | 0 modules/local/process/scflow/dge.nf | 0 modules/local/process/scflow/dirichlet.nf | 0 modules/local/process/scflow/finalize.nf | 0 modules/local/process/scflow/functions.nf | 0 modules/local/process/scflow/integrate.nf | 0 modules/local/process/scflow/ipa.nf | 0 modules/local/process/scflow/mapcelltypes.nf | 0 modules/local/process/scflow/merge.nf | 0 modules/local/process/scflow/mergeqctables.nf | 0 modules/local/process/scflow/plotreddimgenes.nf | 0 modules/local/process/scflow/qc.nf | 0 modules/local/process/scflow/reducedims.nf | 0 modules/local/process/scflow/reportintegrated.nf | 0 modules/local/samplesheet_check.nf | 0 nextflow.config | 0 nextflow_schema.json | 14 ++++++++++---- subworkflows/local/input_check.nf | 0 workflows/scflow.nf | 9 +++++---- 70 files changed, 39 insertions(+), 22 deletions(-) mode change 100644 => 100755 .editorconfig mode change 100644 => 100755 .gitattributes mode change 100644 => 100755 .github/.dockstore.yml mode change 100644 => 100755 .github/CONTRIBUTING.md mode change 100644 => 100755 .github/ISSUE_TEMPLATE/bug_report.md mode change 100644 => 100755 .github/ISSUE_TEMPLATE/config.yml mode change 100644 => 100755 .github/ISSUE_TEMPLATE/feature_request.md mode change 100644 => 100755 .github/PULL_REQUEST_TEMPLATE.md mode change 100644 => 100755 .github/workflows/awsfulltest.yml mode change 100644 => 100755 .github/workflows/awstest.yml mode change 100644 => 100755 .github/workflows/branch.yml mode change 100644 => 100755 .github/workflows/ci.yml mode change 100644 => 100755 .github/workflows/linting.yml mode change 100644 => 100755 .github/workflows/linting_comment.yml mode change 100644 => 100755 .gitignore mode change 100644 => 100755 .markdownlint.yml mode change 100644 => 100755 .nf-core.yml mode change 100644 => 100755 CHANGELOG.md mode change 100644 => 100755 CITATIONS.md mode change 100644 => 100755 CODE_OF_CONDUCT.md mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md mode change 100644 => 100755 assets/NO_FILE.tsv mode change 100644 => 100755 assets/email_template.html mode change 100644 => 100755 assets/email_template.txt mode change 100644 => 100755 assets/nf-core-scflow_logo.png mode change 100644 => 100755 assets/schema_input.json mode change 100644 => 100755 assets/sendmail_template.txt mode change 100644 => 100755 conf/base.config mode change 100644 => 100755 conf/gcp.config mode change 100644 => 100755 conf/igenomes.config mode change 100644 => 100755 conf/modules.config mode change 100644 => 100755 conf/scflow_analysis.config mode change 100644 => 100755 conf/test.config mode change 100644 => 100755 conf/test_full.config mode change 100644 => 100755 docs/README.md mode change 100644 => 100755 docs/images/nf-core-scflow_logo.png mode change 100644 => 100755 docs/images/scflow_workflow.png mode change 100644 => 100755 docs/output.md mode change 100644 => 100755 docs/usage.md mode change 100644 => 100755 lib/nfcore_external_java_deps.jar mode change 100644 => 100755 main.nf mode change 100644 => 100755 modules.json mode change 100644 => 100755 modules/local/functions.nf mode change 100644 => 100755 modules/local/get_software_versions.nf mode change 100644 => 100755 modules/local/process/functions.nf mode change 100644 => 100755 modules/local/process/scflow/checkinputs.nf mode change 100644 => 100755 modules/local/process/scflow/cluster.nf mode change 100644 => 100755 modules/local/process/scflow/dge.nf mode change 100644 => 100755 modules/local/process/scflow/dirichlet.nf mode change 100644 => 100755 modules/local/process/scflow/finalize.nf mode change 100644 => 100755 modules/local/process/scflow/functions.nf mode change 100644 => 100755 modules/local/process/scflow/integrate.nf mode change 100644 => 100755 modules/local/process/scflow/ipa.nf mode change 100644 => 100755 modules/local/process/scflow/mapcelltypes.nf mode change 100644 => 100755 modules/local/process/scflow/merge.nf mode change 100644 => 100755 modules/local/process/scflow/mergeqctables.nf mode change 100644 => 100755 modules/local/process/scflow/plotreddimgenes.nf mode change 100644 => 100755 modules/local/process/scflow/qc.nf mode change 100644 => 100755 modules/local/process/scflow/reducedims.nf mode change 100644 => 100755 modules/local/process/scflow/reportintegrated.nf mode change 100644 => 100755 modules/local/samplesheet_check.nf mode change 100644 => 100755 nextflow.config mode change 100644 => 100755 nextflow_schema.json mode change 100644 => 100755 subworkflows/local/input_check.nf mode change 100644 => 100755 workflows/scflow.nf diff --git a/.editorconfig b/.editorconfig old mode 100644 new mode 100755 diff --git a/.gitattributes b/.gitattributes old mode 100644 new mode 100755 diff --git a/.github/.dockstore.yml b/.github/.dockstore.yml old mode 100644 new mode 100755 diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md old mode 100644 new mode 100755 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md old mode 100644 new mode 100755 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml old mode 100644 new mode 100755 diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md old mode 100644 new mode 100755 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md old mode 100644 new mode 100755 diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.markdownlint.yml b/.markdownlint.yml old mode 100644 new mode 100755 diff --git a/.nf-core.yml b/.nf-core.yml old mode 100644 new mode 100755 diff --git a/CHANGELOG.md b/CHANGELOG.md old mode 100644 new mode 100755 diff --git a/CITATIONS.md b/CITATIONS.md old mode 100644 new mode 100755 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/assets/NO_FILE.tsv b/assets/NO_FILE.tsv old mode 100644 new mode 100755 diff --git a/assets/email_template.html b/assets/email_template.html old mode 100644 new mode 100755 diff --git a/assets/email_template.txt b/assets/email_template.txt old mode 100644 new mode 100755 diff --git a/assets/nf-core-scflow_logo.png b/assets/nf-core-scflow_logo.png old mode 100644 new mode 100755 diff --git a/assets/schema_input.json b/assets/schema_input.json old mode 100644 new mode 100755 diff --git a/assets/sendmail_template.txt b/assets/sendmail_template.txt old mode 100644 new mode 100755 diff --git a/bin/scflow_dge.r b/bin/scflow_dge.r index d419394..6118a21 100755 --- a/bin/scflow_dge.r +++ b/bin/scflow_dge.r @@ -131,7 +131,7 @@ required$add_argument( ) required$add_argument( - "--fc_threshold", + "--logFC_threshold", type = "double", default = 1.1, metavar = "number", @@ -139,7 +139,7 @@ required$add_argument( ) required$add_argument( - "--pval_cutoff", + "--padj_cutoff", type = "double", default = 0.05, metavar = "number", @@ -254,8 +254,6 @@ de_results <- perform_de( ref_class = args$ref_class, confounding_vars = args$confounding_vars, random_effects_var = args$random_effects_var, - fc_threshold = args$fc_threshold, - pval_cutoff = args$pval_cutoff, mast_method = args$mast_method, force_run = args$force_run, ensembl_mapping_file = args$ensembl_mappings, @@ -278,8 +276,8 @@ for (result in names(de_results)) { ) report_de(de_results[[result]], - fc_threshold = args$fc_threshold, - pval_cutoff = args$pval_cutoff, + logFC_threshold = args$logFC_threshold, + padj_cutoff = args$padj_cutoff, n_label = args$n_label, report_folder_path = file.path(getwd()), report_file = paste0(file_name, result, "_scflow_de_report") @@ -289,8 +287,8 @@ for (result in names(de_results)) { p <- scFlow::volcano_plot( dt = de_results[[result]], - fc_threshold = args$fc_threshold, - pval_cutoff = args$pval_cutoff, + logFC_threshold = args$logFC_threshold, + padj_cutoff = args$padj_cutoff, n_label = args$n_label ) diff --git a/bin/scflow_ipa.r b/bin/scflow_ipa.r index 5e09573..39b816c 100755 --- a/bin/scflow_ipa.r +++ b/bin/scflow_ipa.r @@ -64,7 +64,7 @@ required$add_argument( ) required$add_argument( - "--fc_threshold", + "--logFC_threshold", type = "double", default = 1.1, metavar = "number", @@ -72,7 +72,7 @@ required$add_argument( ) required$add_argument( - "--pval_cutoff", + "--padj_cutoff", type = "double", default = 0.05, metavar = "number", @@ -121,8 +121,8 @@ for (gene_file in args$gene_file) { dt <- dt %>% dplyr::filter( - padj <= args$pval_cutoff, - abs(logFC) >= log2(args$fc_threshold) + padj <= args$padj_cutoff, + abs(logFC) >= log2(args$logFC_threshold) ) if (nrow(dt) < 5) { diff --git a/bin/scflow_map_celltypes.r b/bin/scflow_map_celltypes.r index 5c913dd..7ec9204 100755 --- a/bin/scflow_map_celltypes.r +++ b/bin/scflow_map_celltypes.r @@ -53,6 +53,15 @@ required$add_argument( required = TRUE ) +required$add_argument( + "--annotation_level", + default = 1, + type = "integer", + help = "the annotation level of the reference ctd", + required = TRUE, + metavar = "N" +) + required$add_argument( "--species", help = "the biological species (e.g. mouse, human)", @@ -101,6 +110,7 @@ sce <- map_celltypes_sce( ctd_folder = args$ctd_folder, clusters_colname = args$clusters_colname, cells_to_sample = args$cells_to_sample, + annotation_level = as.numeric(args$annotation_level), species = args$species ) diff --git a/bin/scflow_reduce_dims.r b/bin/scflow_reduce_dims.r index 3ed29ae..2788e95 100755 --- a/bin/scflow_reduce_dims.r +++ b/bin/scflow_reduce_dims.r @@ -313,6 +313,7 @@ args <- purrr::map(args, function(x) { sce <- read_sce(args$sce_path, read_metadata = TRUE) +set.seed(42) sce <- reduce_dims_sce( sce, input_reduced_dim = args$input_reduced_dim, diff --git a/conf/base.config b/conf/base.config old mode 100644 new mode 100755 diff --git a/conf/gcp.config b/conf/gcp.config old mode 100644 new mode 100755 diff --git a/conf/igenomes.config b/conf/igenomes.config old mode 100644 new mode 100755 diff --git a/conf/modules.config b/conf/modules.config old mode 100644 new mode 100755 diff --git a/conf/scflow_analysis.config b/conf/scflow_analysis.config old mode 100644 new mode 100755 index 79493ec..591e8f8 --- a/conf/scflow_analysis.config +++ b/conf/scflow_analysis.config @@ -115,6 +115,7 @@ params { // Options: Celltype Annotation cta_clusters_colname = 'clusters' cta_cells_to_sample = 10000 + cta_annotation_level = 2 // Options: Celltype Metrics Report cta_unique_id_var = 'manifest' @@ -137,8 +138,8 @@ params { dge_ref_class = 'Control' dge_confounding_vars = 'cngeneson' // * dge_random_effects_var = 'null' - dge_fc_threshold = 1.1 - dge_pval_cutoff = 0.05 + dge_logFC_threshold = 0.25 + dge_padj_cutoff = 0.05 dge_n_label = 5 dge_force_run = 'false' dge_max_cores = 'null' diff --git a/conf/test.config b/conf/test.config old mode 100644 new mode 100755 diff --git a/conf/test_full.config b/conf/test_full.config old mode 100644 new mode 100755 diff --git a/docs/README.md b/docs/README.md old mode 100644 new mode 100755 diff --git a/docs/images/nf-core-scflow_logo.png b/docs/images/nf-core-scflow_logo.png old mode 100644 new mode 100755 diff --git a/docs/images/scflow_workflow.png b/docs/images/scflow_workflow.png old mode 100644 new mode 100755 diff --git a/docs/output.md b/docs/output.md old mode 100644 new mode 100755 diff --git a/docs/usage.md b/docs/usage.md old mode 100644 new mode 100755 diff --git a/lib/nfcore_external_java_deps.jar b/lib/nfcore_external_java_deps.jar old mode 100644 new mode 100755 diff --git a/main.nf b/main.nf old mode 100644 new mode 100755 diff --git a/modules.json b/modules.json old mode 100644 new mode 100755 diff --git a/modules/local/functions.nf b/modules/local/functions.nf old mode 100644 new mode 100755 diff --git a/modules/local/get_software_versions.nf b/modules/local/get_software_versions.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/functions.nf b/modules/local/process/functions.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/checkinputs.nf b/modules/local/process/scflow/checkinputs.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/cluster.nf b/modules/local/process/scflow/cluster.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/dge.nf b/modules/local/process/scflow/dge.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/dirichlet.nf b/modules/local/process/scflow/dirichlet.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/finalize.nf b/modules/local/process/scflow/finalize.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/functions.nf b/modules/local/process/scflow/functions.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/integrate.nf b/modules/local/process/scflow/integrate.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/ipa.nf b/modules/local/process/scflow/ipa.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/mapcelltypes.nf b/modules/local/process/scflow/mapcelltypes.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/merge.nf b/modules/local/process/scflow/merge.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/mergeqctables.nf b/modules/local/process/scflow/mergeqctables.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/plotreddimgenes.nf b/modules/local/process/scflow/plotreddimgenes.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/qc.nf b/modules/local/process/scflow/qc.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/reducedims.nf b/modules/local/process/scflow/reducedims.nf old mode 100644 new mode 100755 diff --git a/modules/local/process/scflow/reportintegrated.nf b/modules/local/process/scflow/reportintegrated.nf old mode 100644 new mode 100755 diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf old mode 100644 new mode 100755 diff --git a/nextflow.config b/nextflow.config old mode 100644 new mode 100755 diff --git a/nextflow_schema.json b/nextflow_schema.json old mode 100644 new mode 100755 index 8771107..c9a0516 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -826,6 +826,11 @@ "default": 10000, "description": "Max cells to sample." }, + "cta_annotation_level": { + "type": "integer", + "default": 1, + "description": "Annotation level of the reference ctd" + }, "cta_unique_id_var": { "type": "string", "default": "individual", @@ -856,6 +861,7 @@ "required": [ "cta_clusters_colname", "cta_cells_to_sample", + "cta_annotation_level", "cta_unique_id_var", "cta_celltype_var", "cta_facet_vars", @@ -947,13 +953,13 @@ "description": "Random effect confounding variable.", "help_text": "If specified, the term `+ (1 | x ) +`is added to the model, where x is the specified random effects variable." }, - "dge_fc_threshold": { + "dge_logFC_threshold": { "type": "number", "default": 1.1, "description": "Fold-change threshold for plotting.", "help_text": "This absolute fold-change cut-off value is used in plots (e.g. volcano) and the DGE report." }, - "dge_pval_cutoff": { + "dge_padj_cutoff": { "type": "number", "default": 0.05, "description": "Adjusted p-value cutoff.", @@ -993,8 +999,8 @@ "dge_ref_class", "dge_confounding_vars", "dge_random_effects_var", - "dge_fc_threshold", - "dge_pval_cutoff", + "dge_logFC_threshold", + "dge_padj_cutoff", "dge_n_label", "dge_force_run", "dge_max_cores" diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf old mode 100644 new mode 100755 diff --git a/workflows/scflow.nf b/workflows/scflow.nf old mode 100644 new mode 100755 index d79ff84..d62b227 --- a/workflows/scflow.nf +++ b/workflows/scflow.nf @@ -167,6 +167,7 @@ def scflow_mapcelltypes_options = modules['scflow_mapcelltypes'] scflow_mapcelltypes_options.args = "--clusters_colname ${params.cta_clusters_colname} \ --cells_to_sample ${params.cta_cells_to_sample} \ + --annotation_level ${params.cta_annotation_level} \ --species ${params.species} \ --reddimplot_pointsize ${params.reddimplot_pointsize} \ --reddimplot_alpha ${params.reddimplot_alpha}" @@ -198,8 +199,8 @@ scflow_dge_options.args = --ref_class ${params.dge_ref_class} \ --confounding_vars ${params.dge_confounding_vars} \ --random_effects_var ${params.dge_random_effects_var} \ - --pval_cutoff ${params.dge_pval_cutoff} \ - --fc_threshold ${params.dge_fc_threshold} \ + --padj_cutoff ${params.dge_padj_cutoff} \ + --logFC_threshold ${params.dge_logFC_threshold} \ --species ${params.species} \ --max_cores ${params.dge_max_cores}" @@ -214,8 +215,8 @@ scflow_ipa_options.args = "--enrichment_tool ${params.ipa_enrichment_tool} \ --enrichment_method ${params.ipa_enrichment_method} \ --enrichment_database ${params.ipa_enrichment_database} \ - --pval_cutoff ${params.dge_pval_cutoff} \ - --fc_threshold ${params.dge_fc_threshold} \ + --padj_cutoff ${params.dge_padj_cutoff} \ + --logFC_threshold ${params.dge_logFC_threshold} \ --species ${params.species}" def scflow_dirichlet_options = modules['scflow_dirichlet'] From e396e8fa848ef01e0f16cdbc1e26917f47bbb9ca Mon Sep 17 00:00:00 2001 From: nfancy Date: Tue, 18 Apr 2023 16:33:29 +0000 Subject: [PATCH 05/12] added confounding var for dirichlet --- .nf-core.yml | 2 +- bin/scflow_dirichlet.r | 24 +++++++++++++++++++++--- bin/scflow_ipa.r | 31 ++++++++++++++++++++++++++----- conf/scflow_analysis.config | 5 +++-- modules.json | 5 +---- nextflow_schema.json | 8 +++++++- workflows/scflow.nf | 3 ++- 7 files changed, 61 insertions(+), 17 deletions(-) diff --git a/.nf-core.yml b/.nf-core.yml index 17acfd2..e0bcd34 100755 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -9,7 +9,7 @@ lint: schema_params: - input - actions_awsfulltest: False pipeline_todos: False actions_ci: False +repository_type: pipeline diff --git a/bin/scflow_dirichlet.r b/bin/scflow_dirichlet.r index ed0bdee..1791cf7 100755 --- a/bin/scflow_dirichlet.r +++ b/bin/scflow_dirichlet.r @@ -65,11 +65,28 @@ required$add_argument( required = TRUE ) +required$add_argument( + "--confounding_vars", + help = "confounding variables", + metavar = "sex", + required = TRUE +) + # get command line options, if help option encountered print help and exit, # otherwise if options not found on command line then set defaults args <- parser$parse_args() -args$var_order <- strsplit(args$var_order, ",")[[1]] -if (tolower(args$var_order) == "null") { args$var_order <- NULL } + +if (tolower(args$var_order) == "null") { + args$var_order <- NULL + } else { + args$var_order <- strsplit(args$var_order, ",")[[1]] + } + +if (tolower(args$confounding_vars) == "null") { + args$confounding_vars <- NULL + } else { + args$confounding_vars <- strsplit(args$confounding_vars, ",")[[1]] + } # ____________________________________________________________________________ # Start #### @@ -82,7 +99,8 @@ results <- model_celltype_freqs( celltype_var = args$celltype_var, dependent_var = args$dependent_var, ref_class = args$ref_class, - var_order = args$var_order + var_order = args$var_order, + confounding_vars = args$confounding_vars ) ## ............................................................................ diff --git a/bin/scflow_ipa.r b/bin/scflow_ipa.r index 39b816c..b4de18d 100755 --- a/bin/scflow_ipa.r +++ b/bin/scflow_ipa.r @@ -66,7 +66,7 @@ required$add_argument( required$add_argument( "--logFC_threshold", type = "double", - default = 1.1, + default = 0.5, metavar = "number", help = "Absolute fold-change cutoff for DE [default %(default)s]" ) @@ -122,7 +122,7 @@ for (gene_file in args$gene_file) { dt <- dt %>% dplyr::filter( padj <= args$padj_cutoff, - abs(logFC) >= log2(args$logFC_threshold) + abs(logFC) >= args$logFC_threshold ) if (nrow(dt) < 5) { @@ -134,11 +134,32 @@ for (gene_file in args$gene_file) { organism = getOption("scflow_species"), enrichment_tool = args$enrichment_tool, enrichment_method = args$enrichment_method, - enrichment_database = args$enrichment_database, - is_output = TRUE, - output_dir = output_dir + enrichment_database = args$enrichment_database ) + for(i in names(enrichment_result)){ + + output_dir_path <- paste(output_dir, i, sep = "/") + dir.create(output_dir_path) + + res <- enrichment_result[[i]] + + lapply( + setdiff(names(res), c("plot", "metadata")), + function(dt) { + write.table(res[dt], + file = paste(output_dir_path, "/", dt, ".tsv", sep = ""), + row.names = FALSE, + col.names = gsub(dt, "", colnames(res[[dt]])), sep = "\t")}) + lapply( + names(res$plot), + function(p) { + ggplot2::ggsave(paste(output_dir_path, "/", p, ".png", sep = ""), + res$plot[[p]], + device = "png", height = 8, + width = 10, units = "in", dpi = 300)}) + } + if (all(unlist(lapply( enrichment_result, function(dt) { isFALSE(dt$metadata$result) diff --git a/conf/scflow_analysis.config b/conf/scflow_analysis.config index 591e8f8..74f1e40 100755 --- a/conf/scflow_analysis.config +++ b/conf/scflow_analysis.config @@ -115,7 +115,7 @@ params { // Options: Celltype Annotation cta_clusters_colname = 'clusters' cta_cells_to_sample = 10000 - cta_annotation_level = 2 + cta_annotation_level = 2 // Options: Celltype Metrics Report cta_unique_id_var = 'manifest' @@ -154,7 +154,8 @@ params { dirich_celltype_var = 'cluster_celltype' dirich_dependent_var = 'diagnosis' dirich_ref_class = 'Control' - dirich_var_order = 'null' // * + dirich_var_order = 'Control,Case' // * + dirich_confounding_vars = 'null' // * // Options: Plots (Reduced Dim) plotreddim_reduction_methods = 'UMAP_Liger' // * diff --git a/modules.json b/modules.json index 2462038..462d07d 100755 --- a/modules.json +++ b/modules.json @@ -1,8 +1,5 @@ { "name": "nf-core/scflow", "homePage": "https://github.com/nf-core/scflow", - "repos": { - "nf-core/modules": { - } - } + "repos": {} } diff --git a/nextflow_schema.json b/nextflow_schema.json index c9a0516..6ff7652 100755 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1067,6 +1067,11 @@ "default": "Control,Low,High", "description": "Dependent variable classes order.", "help_text": "For plotting and reports, the order of classes for the dependent variable can be manually specified (e.g. 'Control,Low,High')." + }, + "dirich_confounding_vars": { + "type": "string", + "default": "sex", + "description": "Additional confounding variables." } }, "fa_icon": "fas fa-chart-pie", @@ -1075,7 +1080,8 @@ "dirich_celltype_var", "dirich_dependent_var", "dirich_ref_class", - "dirich_var_order" + "dirich_var_order", + "dirich_confounding_vars" ] }, "general_plotting": { diff --git a/workflows/scflow.nf b/workflows/scflow.nf index d62b227..1901648 100755 --- a/workflows/scflow.nf +++ b/workflows/scflow.nf @@ -225,7 +225,8 @@ scflow_dirichlet_options.args = --celltype_var ${params.dirich_celltype_var} \ --dependent_var ${params.dirich_dependent_var} \ --ref_class ${params.dirich_ref_class} \ - --var_order ${params.dirich_var_order}" + --var_order ${params.dirich_var_order} \ + --confounding_vars ${params.dirich_confounding_vars}" def get_software_versions = modules['get_software_versions'] get_software_versions.args = '' From f67842290172a8b07a6be2a79f4129fecfe1079a Mon Sep 17 00:00:00 2001 From: nfancy Date: Tue, 18 Apr 2023 16:36:19 +0000 Subject: [PATCH 06/12] tidied --- bin/scflow_annotate_integrated.R | 85 -------------------------------- 1 file changed, 85 deletions(-) delete mode 100755 bin/scflow_annotate_integrated.R diff --git a/bin/scflow_annotate_integrated.R b/bin/scflow_annotate_integrated.R deleted file mode 100755 index 4e823a6..0000000 --- a/bin/scflow_annotate_integrated.R +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env Rscript -#' Annotate integrated, dims reduced and clustered sce object -# Mahdi Moradi Marjaneh - -# ____________________________________________________________________________ -# Initialization #### - -options(mc.cores = future::availableCores()) - -## ............................................................................ -## Load packages #### -library(argparse) -library(scFlow) - -## ............................................................................ -## Parse command-line arguments #### - -# create parser object -parser <- ArgumentParser() - -# specify options -required <- parser$add_argument_group("Required", "required arguments") -optional <- parser$add_argument_group("Optional", "required arguments") - -required$add_argument( - "--sce_path", - help = "-path to the SingleCellExperiment", - metavar = "dir", - required = TRUE -) - -required$add_argument( - "--categorical_covariates", - help = "-categorical covariates", - metavar = "individual,diagnosis,region,sex", - required = TRUE -) - -### . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .. -### Pre-process args #### - -args <- parser$parse_args() -args <- purrr::map(args, function(x) { -if (length(x) == 1) { -if (toupper(x) == "TRUE") { -return(TRUE) -} -if (toupper(x) == "FALSE") { -return(FALSE) -} -if (toupper(x) == "NULL") { -return(NULL) -} -} -return(x) -}) - -## ............................................................................ -## Annotate integrated sce #### - -sce <- read_sce(args$sce_path) - -sce <- annotate_integrated_sce( -sce, -categorical_covariates = args$categorical_covariates -) - -dir.create(file.path(getwd(), "integration_report")) - -report_integrated_sce( - sce = sce, - report_folder_path = file.path(getwd(), "integration_report"), - report_file = "integrate_reduceDims_cluster_report_scflow", -) - -print("Annotation complete, saving outputs..") - -## ............................................................................ -## Save Outputs #### - -# Save SingleCellExperiment -write_sce( -sce = sce, -folder_path = file.path(getwd(), "integrated_sce") -) From 416c93cb278a7bed9e9e07819c6ee9f35d2e3367 Mon Sep 17 00:00:00 2001 From: nfancy <50167573+nfancy@users.noreply.github.com> Date: Thu, 14 Nov 2024 11:17:23 +0000 Subject: [PATCH 07/12] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f469759..3fe16e5 100755 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Try the pipeline on an in-built, minimal test dataset (all inputs will be automa 3. Download the pipeline and test it on a minimal dataset with a single command: ```console - nextflow run nf-core/scflow -profile test, + nextflow run combiz/nf-core-scflow -r dev-nf -profile test, ``` > - Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. @@ -75,7 +75,7 @@ A complete, automated, scalable, and reproducible case-control analysis can then 1. Start running your own analysis! ```bash - nextflow run nf-core/scflow \ + nextflow run combiz/nf-core-scflow -r dev-nf \ --manifest Manifest.tsv \ --input Samplesheet.tsv \ -c scflow_params.config \ From 7a490a2734168dcaf6d583fcda7662ea93920fcc Mon Sep 17 00:00:00 2001 From: nfancy <50167573+nfancy@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:04:52 +0000 Subject: [PATCH 08/12] Update base.config --- conf/base.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/base.config b/conf/base.config index 3b0559d..2f7100d 100755 --- a/conf/base.config +++ b/conf/base.config @@ -40,7 +40,7 @@ process { time = { check_max( 8.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 72.GB * task.attempt, 'memory' ) } time = { check_max( 16.h * task.attempt, 'time' ) } } @@ -48,7 +48,9 @@ process { time = { check_max( 20.h * task.attempt, 'time' ) } } withLabel:process_high_memory { + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 200.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } } withLabel:error_ignore { errorStrategy = 'ignore' From aa4c75fb19166a7f2bcc7a3d822f50631bad5578 Mon Sep 17 00:00:00 2001 From: nfancy <50167573+nfancy@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:29:11 +0000 Subject: [PATCH 09/12] Add files via upload --- conf/imperial.config | 113 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 conf/imperial.config diff --git a/conf/imperial.config b/conf/imperial.config new file mode 100644 index 0000000..134b354 --- /dev/null +++ b/conf/imperial.config @@ -0,0 +1,113 @@ +//Profile config names for nf-core/configs + +params { + // Config Params + config_profile_description = 'Imperial College London - HPC Profile -- provided by nf-core/configs.' + config_profile_contact = 'Nurun Fancy (@nfancy)' + config_profile_url = 'https://www.imperial.ac.uk/admin-services/ict/self-service/research-support/rcs/' + + // Resources + max_memory = 920.GB + max_cpus = 256 + max_time = 1000.h +} + +process { + resourceLimits = [ + memory: 920.GB, + cpus: 256, + time: 1000.h + ] +} + +profiles { + imperial { + process { + executor = 'pbspro' + + // Update amount of max retries and set "retry" as the error strategy for all error codes + errorStrategy = 'retry' + maxRetries = 5 + maxErrors = '-1' + + + // General resource requirements + cpus = { 1 * task.attempt } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } + + // Process-specific resource requirements + withLabel: process_single { + cpus = 1 + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } + } + + withLabel: process_low { + cpus = { 2 * task.attempt } + memory = { 48.GB * task.attempt } + time = { 8.h * task.attempt } + } + + withLabel: process_medium { + cpus = { 8 * task.attempt } + memory = { 64.GB * task.attempt } + time = { 12.h * task.attempt } + } + + withLabel: process_high { + cpus = { 12 * task.attempt } + memory = { 120.GB * task.attempt } + time = { 12.h * task.attempt } + } + + withLabel: process_long { + cpus = 9 + memory = 100.GB + time = { 24.h * task.attempt } + } + + withLabel: process_high_memory { + cpus = { 10 * task.attempt } + memory = { 200.GB * task.attempt } + time = { 24.h * task.attempt } + } + withLabel: with_gpus { + time = 24.h + clusterOptions = '-l select=1:ncpus=4:mem=24gb:ngpus=1:gpu_type=RTX6000' + maxForks = 1 + containerOptions = { + workflow.containerEngine == "singularity" + ? '--nv --env CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES' + : (workflow.containerEngine == "docker" ? '--gpus all' : null) + } + beforeScript = 'module load tools/prod' + } + } + } + medbio { + process { + executor = 'pbspro' + } + } +} + +executor { + '$pbspro' { + queueSize = 49 + submitRateLimit = '10 sec' + maxForks = 49 + } + + '$local' { + cpus = 2 + queueSize = 1 + memory = '6 GB' + } +} + +singularity { + enabled = true + autoMounts = true + runOptions = "-B /rds/,/rds/general/user/${USER}/ephemeral/tmp/:/tmp,/rds/general/user/${USER}/ephemeral/tmp/:/var/tmp" +} From 23ae71b1dcbc547f33780e7aaf89ef4d03edafdc Mon Sep 17 00:00:00 2001 From: nfancy <50167573+nfancy@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:31:34 +0000 Subject: [PATCH 10/12] Update imperial.config --- conf/imperial.config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/imperial.config b/conf/imperial.config index 134b354..f6ccd7c 100644 --- a/conf/imperial.config +++ b/conf/imperial.config @@ -27,7 +27,7 @@ profiles { // Update amount of max retries and set "retry" as the error strategy for all error codes errorStrategy = 'retry' - maxRetries = 5 + maxRetries = 1 maxErrors = '-1' @@ -56,9 +56,9 @@ profiles { } withLabel: process_high { - cpus = { 12 * task.attempt } - memory = { 120.GB * task.attempt } - time = { 12.h * task.attempt } + cpus = { 8 * task.attempt } + memory = { 100.GB * task.attempt } + time = { 16.h * task.attempt } } withLabel: process_long { @@ -68,9 +68,9 @@ profiles { } withLabel: process_high_memory { - cpus = { 10 * task.attempt } + cpus = { 8 * task.attempt } memory = { 200.GB * task.attempt } - time = { 24.h * task.attempt } + time = { 8.h * task.attempt } } withLabel: with_gpus { time = 24.h From a703d8508b858105e191f9ec21a5b2408ce20d47 Mon Sep 17 00:00:00 2001 From: nfancy <50167573+nfancy@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:33:42 +0000 Subject: [PATCH 11/12] Update imperial.config --- conf/imperial.config | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/conf/imperial.config b/conf/imperial.config index f6ccd7c..5ef34e0 100644 --- a/conf/imperial.config +++ b/conf/imperial.config @@ -85,18 +85,13 @@ profiles { } } } - medbio { - process { - executor = 'pbspro' - } - } } executor { '$pbspro' { queueSize = 49 submitRateLimit = '10 sec' - maxForks = 49 + maxForks = 45 } '$local' { From 53f3e4a20e148b245bd570e4beffafcba40e81e0 Mon Sep 17 00:00:00 2001 From: nfancy <50167573+nfancy@users.noreply.github.com> Date: Tue, 19 Nov 2024 21:20:49 +0000 Subject: [PATCH 12/12] Delete conf/imperial.config --- conf/imperial.config | 108 ------------------------------------------- 1 file changed, 108 deletions(-) delete mode 100644 conf/imperial.config diff --git a/conf/imperial.config b/conf/imperial.config deleted file mode 100644 index 5ef34e0..0000000 --- a/conf/imperial.config +++ /dev/null @@ -1,108 +0,0 @@ -//Profile config names for nf-core/configs - -params { - // Config Params - config_profile_description = 'Imperial College London - HPC Profile -- provided by nf-core/configs.' - config_profile_contact = 'Nurun Fancy (@nfancy)' - config_profile_url = 'https://www.imperial.ac.uk/admin-services/ict/self-service/research-support/rcs/' - - // Resources - max_memory = 920.GB - max_cpus = 256 - max_time = 1000.h -} - -process { - resourceLimits = [ - memory: 920.GB, - cpus: 256, - time: 1000.h - ] -} - -profiles { - imperial { - process { - executor = 'pbspro' - - // Update amount of max retries and set "retry" as the error strategy for all error codes - errorStrategy = 'retry' - maxRetries = 1 - maxErrors = '-1' - - - // General resource requirements - cpus = { 1 * task.attempt } - memory = { 6.GB * task.attempt } - time = { 4.h * task.attempt } - - // Process-specific resource requirements - withLabel: process_single { - cpus = 1 - memory = { 6.GB * task.attempt } - time = { 4.h * task.attempt } - } - - withLabel: process_low { - cpus = { 2 * task.attempt } - memory = { 48.GB * task.attempt } - time = { 8.h * task.attempt } - } - - withLabel: process_medium { - cpus = { 8 * task.attempt } - memory = { 64.GB * task.attempt } - time = { 12.h * task.attempt } - } - - withLabel: process_high { - cpus = { 8 * task.attempt } - memory = { 100.GB * task.attempt } - time = { 16.h * task.attempt } - } - - withLabel: process_long { - cpus = 9 - memory = 100.GB - time = { 24.h * task.attempt } - } - - withLabel: process_high_memory { - cpus = { 8 * task.attempt } - memory = { 200.GB * task.attempt } - time = { 8.h * task.attempt } - } - withLabel: with_gpus { - time = 24.h - clusterOptions = '-l select=1:ncpus=4:mem=24gb:ngpus=1:gpu_type=RTX6000' - maxForks = 1 - containerOptions = { - workflow.containerEngine == "singularity" - ? '--nv --env CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES' - : (workflow.containerEngine == "docker" ? '--gpus all' : null) - } - beforeScript = 'module load tools/prod' - } - } - } -} - -executor { - '$pbspro' { - queueSize = 49 - submitRateLimit = '10 sec' - maxForks = 45 - } - - '$local' { - cpus = 2 - queueSize = 1 - memory = '6 GB' - } -} - -singularity { - enabled = true - autoMounts = true - runOptions = "-B /rds/,/rds/general/user/${USER}/ephemeral/tmp/:/tmp,/rds/general/user/${USER}/ephemeral/tmp/:/var/tmp" -}