Merge branch 'development' of https://github.com/kircherlab/MPRAsnake…

…flow into feat/update_snakemake

.gitignore

@@ -7,6 +7,9 @@ logs
 !config/*
 !resources
 !resources/**
+resources/**/.local
+resources/**/.cache
+resources/**/.ipython
 !workflow
 !workflow/**
 !.gitattributes
@@ -27,4 +30,4 @@ mix_data
 *report.html
 *.simg
 *results
-.DS_Store
+.DS_Store

workflow/Snakefile

@@ -134,6 +134,15 @@ rule all:
                 "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged.combined.tsv.gz",
                 "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged.combined.tsv.gz",
                 "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged_barcode_assigned_counts.tsv.gz",
+                "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged_barcode_assigned_counts.tsv.gz",
+                "results/experiments/{project}/reporter_experiment.oligo.{condition}.{assignment}.{config}.all.tsv.gz",
+                "results/experiments/{project}/reporter_experiment.barcode.{condition}.{assignment}.{config}.all.tsv.gz",
+            ]
+        ),
+        getOutputProjectConditionAssignmentConfigThreshold_helper(
+            [
+                "results/experiments/{project}/reporter_experiment.barcode.{condition}.{assignment}.{config}.min_oligo_threshold_{threshold}.tsv.gz",
+                "results/experiments/{project}/reporter_experiment.oligo.{condition}.{assignment}.{config}.min_oligo_threshold_{threshold}.tsv.gz",
             ]
         ),
         # assignment statistic
@@ -263,6 +272,15 @@ rule all_experiments:
                 "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged.combined.tsv.gz",
                 "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged.combined.tsv.gz",
                 "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged_barcode_assigned_counts.tsv.gz",
+                "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged_barcode_assigned_counts.tsv.gz",
+                "results/experiments/{project}/reporter_experiment.oligo.{condition}.{assignment}.{config}.all.tsv.gz",
+                "results/experiments/{project}/reporter_experiment.barcode.{condition}.{assignment}.{config}.all.tsv.gz",
+            ]
+        ),
+        getOutputProjectConditionAssignmentConfigThreshold_helper(
+            [
+                "results/experiments/{project}/reporter_experiment.barcode.{condition}.{assignment}.{config}.min_oligo_threshold_{threshold}.tsv.gz",
+                "results/experiments/{project}/reporter_experiment.oligo.{condition}.{assignment}.{config}.min_oligo_threshold_{threshold}.tsv.gz",
             ]
         ),
         # assignment statistic

workflow/rules/assigned_counts.smk

@@ -67,7 +67,9 @@ rule assigned_counts_assignBarcodes:
         script=getScript("count/merge_BC_and_assignment.py"),
     output:
         counts="results/experiments/{project}/assigned_counts/{assignment}/{condition}_{replicate}_{type}_final_counts.config.{config}.tsv.gz",
-        stats="results/experiments/{project}/statistic/assigned_counts/{assignment}/{condition}_{replicate}_{type}_{config}.statistic.tsv.gz",
+        statistic=temp(
+            "results/experiments/{project}/statistic/assigned_counts/{assignment}/{condition}_{replicate}_{type}_{config}.statistic.tsv.gz"
+        ),
     params:
         name="{condition}_{replicate}_{type}",
     log:
@@ -79,7 +81,7 @@ rule assigned_counts_assignBarcodes:
         python {input.script} --counts {input.counts} \
         --assignment {input.association} \
         --output {output.counts} \
-        --statistic {output.stats} \
+        --statistic {output.statistic} \
         --name {params.name} &> {log}
         """
 
@@ -97,7 +99,9 @@ rule assigned_counts_dna_rna_merge:
     output:
         counts="results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_{replicate}_merged_assigned_counts.tsv.gz",
         bc_counts="results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_{replicate}_barcode_assigned_counts.tsv.gz",
-        stats="results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/{condition}_{replicate}_merged_assigned_counts.statistic.tsv.gz",
+        statistic=temp(
+            "results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/{condition}_{replicate}_merged_assigned_counts.statistic.tsv.gz"
+        ),
     params:
         minRNACounts=lambda wc: config["experiments"][wc.project]["configs"][
             wc.config
@@ -116,7 +120,7 @@ rule assigned_counts_dna_rna_merge:
         --assignment {input.association} \
         --output {output.counts} \
         --bcOutput {output.bc_counts} \
-        --statistic {output.stats} &> {log}
+        --statistic {output.statistic} &> {log}
         """
 
 
@@ -137,7 +141,6 @@ rule assigned_counts_make_master_tables:
         all="results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged.tsv.gz",
         thresh="results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged.tsv.gz",
     params:
-        cond="{condition}",
         files=lambda wc: ",".join(
             expand(
                 "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_{replicate}_merged_assigned_counts.tsv.gz",
@@ -161,7 +164,6 @@ rule assigned_counts_make_master_tables:
     shell:
         """
         Rscript {input.script} \
-        --condition {params.cond} \
         --threshold {params.thresh} \
         --files {params.files} \
         --replicates {params.replicates} \
@@ -185,7 +187,8 @@ rule assigned_counts_combine_replicates_barcode_output:
         ),
         script=getScript("count/merge_replicates_barcode_counts.py"),
     output:
-        bc_merged="results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged_barcode_assigned_counts.tsv.gz",
+        bc_merged_thresh="results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged_barcode_assigned_counts.tsv.gz",
+        bc_merged_all="results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged_barcode_assigned_counts.tsv.gz",
     params:
         thresh=lambda wc: config["experiments"][wc.project]["configs"][wc.config][
             "filter"
@@ -218,7 +221,8 @@ rule assigned_counts_combine_replicates_barcode_output:
         python {input.script} {params.bc_counts} \
         --threshold {params.thresh} \
         {params.replicates}  \
-        --output {output.bc_merged} &> {log}
+        --output-threshold {output.bc_merged_thresh} \
+        --output {output.bc_merged_all} &> {log}
         """
 
 
@@ -250,3 +254,49 @@ rule assigned_counts_combine_replicates:
         {params.label_file} \
         --output {output}  &> {log}
         """
+
+
+rule assigned_counts_copy_final_all_files:
+    """
+    Will copy final files to the main folder so that it is creal which files to use.
+    """
+    conda:
+        "../envs/default.yaml"
+    input:
+        all=lambda wc: "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged.tsv.gz",
+        bc_all=lambda wc: "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_merged_barcode_assigned_counts.tsv.gz",
+    output:
+        all="results/experiments/{project}/reporter_experiment.oligo.{condition}.{assignment}.{config}.all.tsv.gz",
+        bc_all="results/experiments/{project}/reporter_experiment.barcode.{condition}.{assignment}.{config}.all.tsv.gz",
+    log:
+        temp(
+            "results/logs/assigned_counts/copy_final_all_files.{project}.{condition}.{assignment}.{config}.log"
+        ),
+    shell:
+        """
+        cp {input.all} {output.all} &> {log}
+        cp {input.bc_all} {output.bc_all} &>> {log}
+        """
+
+
+rule assigned_counts_copy_final_thresh_files:
+    """
+    Will copy final files to the main folder so that it is creal which files to use.
+    """
+    conda:
+        "../envs/default.yaml"
+    input:
+        thresh=lambda wc: "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged.tsv.gz",
+        bc_thresh=lambda wc: "results/experiments/{project}/assigned_counts/{assignment}/{config}/{condition}_allreps_minThreshold_merged_barcode_assigned_counts.tsv.gz",
+    output:
+        thresh="results/experiments/{project}/reporter_experiment.oligo.{condition}.{assignment}.{config}.min_oligo_threshold_{threshold}.tsv.gz",
+        bc_thresh="results/experiments/{project}/reporter_experiment.barcode.{condition}.{assignment}.{config}.min_oligo_threshold_{threshold}.tsv.gz",
+    log:
+        temp(
+            "results/logs/assigned_counts/copy_final_thresh_files.{project}.{condition}.{assignment}.{config}.{threshold}.log"
+        ),
+    shell:
+        """
+        cp {input.thresh} {output.thresh} &> {log}
+        cp {input.bc_thresh} {output.bc_thresh} &>> {log}
+        """

workflow/rules/assignment.smk

@@ -248,7 +248,7 @@ rule assignment_filter:
         python {input.script} \
         -m {params.min_support} -f {params.fraction} {params.unknown_other} {params.ambiguous} | \
         tee >(gzip -c > {output.ambigous}) | \
-        awk -v "OFS=\\t"  -F"\\t" '{{ if (($2 != \"ambiguous\") && ($2 != \"other\")) {{ print $0 }} }}' | \
+        awk -v "OFS=\\t"  -F"\\t" '{{ if (($2 != \"ambiguous\") && ($2 != \"other\")) {{ print $1,$2 }} }}' | \
         gzip -c > {output.final} 2> {log.err};
         gzip -l {output.final} | awk 'NR==2 {{exit($2==0)}}' || {{ echo "Error: Empty barcode file {output.final}. No barcodes detected!" >> {log.err}; exit 1; }}
         """

workflow/rules/assignment/statistic.smk

@@ -48,7 +48,7 @@ rule assignment_statistic_assignment:
     conda:
         "../../envs/r.yaml"
     input:
-        bc="results/assignment/{assignment}/assignment_barcodes.{assignment_config}.tsv.gz",
+        bc="results/assignment/{assignment}/assignment_barcodes_with_ambigous.{assignment_config}.tsv.gz",
         script=getScript("assignment/statistic_assignment.R"),
     output:
         stats="results/assignment/{assignment}/statistic/assignment.{assignment_config}.tsv.gz",

workflow/rules/common.smk

@@ -374,6 +374,34 @@ def getOutputProjectConditionAssignmentConfig_helper(files):
     return output
 
 
+def getOutputProjectConditionAssignmentConfigThreshold_helper(files):
+    """
+    Inserts {project}, {condition}, {assignment} {config} (from configs of project) and Threshold from config into given file.
+    """
+    output = []
+    projects = getProjects()
+    for project in projects:
+        try:
+            conditions = getConditions(project)
+            for condition in conditions:
+                for conf in getConfigs(project):
+                    threshold = config["experiments"][project]["configs"][conf][
+                        "filter"
+                    ]["bc_threshold"]
+                    for file in files:
+                        output += expand(
+                            file,
+                            project=project,
+                            condition=condition,
+                            assignment=getProjectAssignments(project),
+                            config=conf,
+                            threshold=threshold,
+                        )
+        except MissingAssignmentInConfigException:
+            continue
+    return output
+
+
 def getOutputProjectAssignmentConfig_helper(files, betweenReplicates=False):
     """
     Inserts {project}, {assignment} and {config} (from configs of project) from config into given file.

workflow/rules/statistic/assigned_counts.smk

@@ -86,7 +86,9 @@ rule statistic_assigned_counts_combine_stats_dna_rna_merge:
         ),
         script=getScript("count/merge_statistic_tables.py"),
     output:
-        "results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/combined/{condition}_merged_assigned_counts.statistic.tsv.gz",
+        temp(
+            "results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/combined/{condition}_merged_assigned_counts.statistic.tsv.gz"
+        ),
     params:
         cond="{condition}",
         statistic=lambda wc: " ".join(

workflow/rules/statistic/bc_overlap.smk

@@ -50,7 +50,7 @@ rule statistic_bc_overlap_combine_counts:
     conda:
         "../../envs/default.yaml"
     input:
-        stats=lambda wc: expand(
+        statistic=lambda wc: expand(
             "results/experiments/{{project}}/statistic/bc_overlap/counts/overlapBCandCounts.{condition}_{type}.{config}.tsv",
             type=["DNA", "RNA"],
             condition=getConditions(wc.project),
@@ -74,8 +74,8 @@ rule statistic_bc_overlap_combine_counts:
         """
         set +o pipefail;
         (
-            cat {input.stats[0]} | head -n 1;
-            for i in {input.stats}; do
+            cat {input.statistic[0]} | head -n 1;
+            for i in {input.statistic}; do
                 cat $i | tail -n +2
             done;
         ) > {output} 2> {log}
@@ -86,7 +86,7 @@ rule statistic_bc_overlap_combine_assigned_counts:
     conda:
         "../../envs/default.yaml"
     input:
-        stats=lambda wc: expand(
+        statistic=lambda wc: expand(
             "results/experiments/{{project}}/statistic/bc_overlap/assigned_counts/{{assignment}}/overlapBCandCounts.{condition}_{type}.{{config}}.tsv",
             type=["DNA", "RNA"],
             condition=getConditions(wc.project),
@@ -111,8 +111,8 @@ rule statistic_bc_overlap_combine_assigned_counts:
         """
         set +o pipefail;
         (
-            cat {input.stats[0]} | head -n 1;
-            for i in {input.stats}; do
+            cat {input.statistic[0]} | head -n 1;
+            for i in {input.statistic}; do
                 cat $i | tail -n +2
             done;
         ) > {output} 2> {log}

workflow/rules/statistic/correlation.smk

@@ -45,7 +45,9 @@ rule statistic_correlation_bc_counts:
                 "Plot": "Ratio",
             },
         ),
-        "results/experiments/{project}/statistic/barcode/{raw_or_assigned}/{condition}_{config}_barcode_correlation.tsv",
+        temp(
+            "results/experiments/{project}/statistic/barcode/{raw_or_assigned}/{condition}_{config}_barcode_correlation.tsv"
+        ),
     params:
         replicates=lambda wc: ",".join(
             getMergedCounts(wc.project, wc.raw_or_assigned, wc.condition, wc.config)[1]
@@ -287,8 +289,12 @@ rule statistic_correlation_calculate:
                 ),
             },
         ),
-        "results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/{condition}_correlation.tsv",
-        "results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/{condition}_correlation_minThreshold.tsv",
+        temp(
+            "results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/{condition}_correlation.tsv"
+        ),
+        temp(
+            "results/experiments/{project}/statistic/assigned_counts/{assignment}/{config}/{condition}_correlation_minThreshold.tsv"
+        ),
     params:
         cond="{condition}",
         files=lambda wc: ",".join(