Use mem_max for all merging and sync changes from main

bin/post_process_sce.R

@@ -279,5 +279,11 @@ if (length(reducedDimNames(processed_sce)) == 0) {
 # write out  filtered SCE with additional filtering column
 readr::write_rds(sce, opt$out_filtered_sce_file, compress = "bz2")
 
-# write out processed SCE
-readr::write_rds(processed_sce, opt$out_processed_sce_file, compress = "bz2")
+# only write out processed SCE if > 0 cells
+if (ncol(processed_sce) > 0) {
+  # write out processed SCE
+  readr::write_rds(processed_sce, opt$out_processed_sce_file, compress = "bz2")
+} else {
+  # make an empty processed file
+  file.create(opt$out_processed_sce_file)
+}

bin/sce_qc_report.R

@@ -150,7 +150,13 @@ if (opt$workflow_commit == "null") {
 # read sce files
 unfiltered_sce <- readr::read_rds(opt$unfiltered_sce)
 filtered_sce <- readr::read_rds(opt$filtered_sce)
-processed_sce <- readr::read_rds(opt$processed_sce)
+
+# make sure processed sce has an object, otherwise set to NULL
+if (file.size(opt$processed_sce) > 0) {
+  processed_sce <- readr::read_rds(opt$processed_sce)
+} else {
+  processed_sce <- NULL
+}
 
 # Compile metadata for output files
 sce_meta <- metadata(unfiltered_sce)

config/process_base.config

@@ -24,6 +24,9 @@ process {
   withLabel: mem_96 {
     memory = {check_memory(48.GB  + 48.GB * task.attempt, params.max_memory)}
   }
+  withLabel: mem_max {
+    memory = {task.attempt > 1 ? params.max_memory : check_memory(96.GB, params.max_memory)}
+  }
   withLabel: cpus_2  {
     cpus = {check_cpus(2, params.max_cpus)}
   }

external-instructions.md

@@ -86,12 +86,12 @@ Using the above command will run the workflow from the `main` branch of the work
 To update to the latest released version you can run `nextflow pull AlexsLemonade/scpca-nf` before the `nextflow run` command.
 
 To be sure that you are using a consistent version, you can specify use of a release tagged version of the workflow, set below with the `-r` flag.
-The command below will pull the `scpca-nf` workflow directly from Github using the `v0.7.2` version.
+The command below will pull the `scpca-nf` workflow directly from Github using the `v0.7.3` version.
 Released versions can be found on the [`scpca-nf` repository releases page](https://github.com/AlexsLemonade/scpca-nf/releases).
 
 ```sh
 nextflow run AlexsLemonade/scpca-nf \
-  -r v0.7.2 \
+  -r v0.7.3 \
   -config <path to config file>  \
   -profile <name of profile>
 ```
@@ -325,7 +325,7 @@ If you will be analyzing spatial expression data, you will also need the Cell Ra
 
 If your compute nodes do not have internet access, you will likely have to pre-pull the required container images as well.
 When doing this, it is important to be sure that you also specify the revision (version tag) of the `scpca-nf` workflow that you are using.
-For example, if you would run `nextflow run AlexsLemonade/scpca-nf -r v0.7.2`, then you will want to set `-r v0.7.2` for `get_refs.py` as well to be sure you have the correct containers.
+For example, if you would run `nextflow run AlexsLemonade/scpca-nf -r v0.7.3`, then you will want to set `-r v0.7.3` for `get_refs.py` as well to be sure you have the correct containers.
 By default, `get_refs.py` will download files and images associated with the latest release.
 
 If your system uses Docker, you can add the `--docker` flag:

internal-instructions.md

@@ -87,7 +87,7 @@ Please refer to our [`CONTRIBUTING.md`](CONTRIBUTING.md#stub-workflows) for more
 When running the workflow for a project or group of samples that is ready to be released on ScPCA portal, please use the tag for the latest release:
 
 ```
-nextflow run AlexsLemonade/scpca-nf -r v0.7.2 -profile ccdl,batch --project SCPCP000000
+nextflow run AlexsLemonade/scpca-nf -r v0.7.3 -profile ccdl,batch --project SCPCP000000
 ```
 
 ### Processing example data

main.nf

@@ -261,8 +261,22 @@ workflow {
   all_sce_ch = sce_ch.no_genetic.mix(genetic_demux_sce.out)
   post_process_sce(all_sce_ch)
 
+
+  post_process_ch = post_process_sce.out
+    // only continue processing any samples with > 0 cells left after processing
+    .branch{
+      continue_processing: it[3].size() > 0
+      skip_processing: true
+      }
+
+  // send library ids in post_process_ch.skip_processing to log
+  post_process_ch.skip_processing
+    .subscribe{
+      log.error("There are no cells found in the processed object for ${it[0].library_id}.")
+    }
+
   // Cluster SCE
-  cluster_sce(post_process_sce.out)
+  cluster_sce(post_process_ch.continue_processing)
 
   if (params.perform_celltyping) {
     // Perform celltyping, if specified
@@ -271,6 +285,9 @@ workflow {
     annotated_celltype_ch = cluster_sce.out
   }
 
+  // combine back with libraries that skipped post processing
+  sce_output_ch = annotated_celltype_ch.mix(post_process_ch.skip_processing)
+
   // generate QC reports
   sce_qc_report(
     annotated_celltype_ch,

merge.nf

@@ -29,7 +29,7 @@ if (param_error) {
 // merge individual SCE objects into one SCE object
 process merge_sce {
   container params.SCPCATOOLS_CONTAINER
-  label 'mem_32'
+  label 'mem_max'
   publishDir "${params.results_dir}/${merge_group_id}/merged"
   input:
     tuple val(merge_group_id), val(has_adt), val(library_ids), path(scpca_nf_file)
@@ -60,7 +60,7 @@ process merge_sce {
 process generate_merge_report {
   container params.SCPCATOOLS_CONTAINER
   publishDir "${params.results_dir}/${merge_group_id}/merged"
-  label 'mem_16'
+  label 'mem_max'
   input:
     tuple path(merged_sce_file), val(merge_group_id), val(has_adt)
     path(report_template)
@@ -86,7 +86,7 @@ process generate_merge_report {
 
 process export_anndata {
     container params.SCPCATOOLS_CONTAINER
-    label 'mem_32'
+    label 'mem_max'
     tag "${merge_group_id}"
     publishDir "${params.results_dir}/${merge_group_id}/merged", mode: 'copy'
     input:

modules/classify-celltypes.nf

@@ -49,7 +49,7 @@ process classify_cellassign {
       mode: 'copy',
       pattern: "${cellassign_dir}"
     )
-  label 'mem_96'
+  label 'mem_max'
   label 'cpus_12'
   tag "${meta.library_id}"
   input:

nextflow.config

@@ -5,7 +5,7 @@ manifest {
   homePage = 'https://github.com/AlexsLemonade/scpca-nf'
   mainScript = 'main.nf'
   defaultBranch = 'main'
-  version = 'v0.7.2'
+  version = 'v0.7.3'
 }
 
 // global parameters for workflows