PD-2721: Add checks for empty cells (#1353)

pipeline_versions.txt

@@ -4,7 +4,7 @@ Multiome	 5.5.0	2024-08-06
 PairedTag	 1.5.0	2024-08-06 
 atac	 2.2.3	2024-08-02 
 SlideSeq	 3.4.0	2024-08-06 
-snm3C	 4.0.3	2024-08-05 
+snm3C	 4.0.4	2024-08-06 
 MultiSampleSmartSeq2SingleNucleus	 1.4.2	2024-08-25-02 
 scATAC	 1.3.2	2023-08-03 
 SmartSeq2SingleSample	 5.1.20	2023-04-19 

pipelines/skylab/snm3C/snm3C.changelog.md

@@ -1,7 +1,12 @@
+# 4.0.4
+2024-08-06 (Date of Last Commit)
+
+* Updated the Demultiplexing task in the snm3C wdl to flag when file/cell is empty
+
 # 4.0.3
-2024-08-05 (Date of Last Commit)
+2024-08-06 (Date of Last Commit)
 
-* Updated the demultiplexing task in snm3C wdl to dynamically update the batch number based on the number of fastq files present
+* Updated the Demultiplexing task in snm3C wdl to dynamically update the batch number based on the number of fastq files present
 
 # 4.0.2
 2024-07-09 (Date of Last Commit)

pipelines/skylab/snm3C/snm3C.wdl

@@ -44,7 +44,7 @@ workflow snm3C {
     }
 
     # version of the pipeline
-    String pipeline_version = "4.0.3"
+    String pipeline_version = "4.0.4"
 
     call Demultiplexing {
         input:
@@ -154,6 +154,8 @@ task Demultiplexing {
     File random_primer_indexes
     String plate_id
     Int batch_number
+    Int min_threshold = 100
+    Int max_threshold = 10000000
     String docker
 
     Int disk_size = 1000
@@ -179,7 +181,7 @@ task Demultiplexing {
     $WORKING_DIR/r2.fastq.gz \
     > $WORKING_DIR/~{plate_id}.stats.txt
 
-    # remove the fastq files that end in unknown-R1.fq.gz and unknown-R2.fq.gz
+    # Remove the fastq files that end in unknown-R1.fq.gz and unknown-R2.fq.gz
     rm $WORKING_DIR/*-unknown-R{1,2}.fq.gz
 
     python3 <<CODE
@@ -199,27 +201,31 @@ task Demultiplexing {
         trimmed_count = int(adapter_match[1])
         adapter_counts[adapter_name] = trimmed_count
 
-    # Removing fastq files with trimmed reads greater than 30
-    threshold = 10000000
-
+    # Removing fastq files with trimmed reads greater than 10000000 or less than 100
     for filename in os.listdir(working_dir):
         if filename.endswith('.fq.gz'):
             file_path = os.path.join(working_dir, filename)
-            adapter_name = re.search(r'A(\d+)-R', filename)
+            adapter_name = re.search(r'([A-Za-z]\d+)-R', filename).group(1)
             if adapter_name:
-                adapter_name = 'A' + adapter_name.group(1)
-                if adapter_name in adapter_counts and adapter_counts[adapter_name] > threshold:
-                    os.remove(file_path)
+                if adapter_name in adapter_counts:
+                    if adapter_counts[adapter_name] < ~{min_threshold} or adapter_counts[adapter_name] > ~{max_threshold}:
+                        print("Removing ", file_path, " with count equal to ", adapter_counts[adapter_name])
+                        os.remove(file_path)
     CODE
+    
+    # Check if the number of *R1.fq.gz files is 0
+    if [[ $(ls | grep "\-R1.fq.gz" | wc -l) -eq 0 ]]; then
+        echo "Error: No files found. All fastq files were removed. Exiting."
+        exit 1
+    fi
 
     # Batch the fastq files into folders of batch_number size
     R1_files=($(ls $WORKING_DIR | grep "\-R1.fq.gz"))
     R2_files=($(ls $WORKING_DIR | grep "\-R2.fq.gz"))
+    batch_number=~{batch_number}
     total_files=${#R1_files[@]}
     echo "Total files: $total_files"
 
-    batch_number=~{batch_number}
-
     if [[ $total_files -lt $batch_number ]]; then
         echo "Warning: Number of files is less than the batch number. Updating batch number to $total_files."
         batch_number=$total_files
@@ -229,14 +235,14 @@ task Demultiplexing {
         mkdir -p "batch${i}"  # Combine batch and i, use -p to create parent dirs
     done
 
-    # Counter for the folder index
+    # Counter for the folder index and create emptycells file
     folder_index=1
-    WORKING_DIR=`pwd`
 
     # Distribute the FASTQ files and create TAR files
     for file in "${R1_files[@]}"; do
         sample_id=$(basename "$file" "-R1.fq.gz")
         r2_file="${sample_id}-R2.fq.gz"
+         
         mv $WORKING_DIR/$file batch$((folder_index))/$file
         mv $WORKING_DIR/$r2_file batch$((folder_index))/$r2_file
         # Increment the counter
@@ -249,7 +255,7 @@ task Demultiplexing {
         tar -cf - $WORKING_DIR/batch${i}/*.fq.gz | pigz > ~{plate_id}.${i}.cutadapt_output_files.tar.gz
     done
   >>>
-
+  
   runtime {
     docker: docker
     disks: "local-disk ${disk_size} SSD"