Compatibility with custom dataset

.gitignore

@@ -11,4 +11,5 @@ test/grmmatrix
 LocalPreferences.toml
 test/sandbox.jl
 docs/Manifest.toml
-*.DS_Store
+*.DS_Store
+trace.txt*

conf/ci_jobs/from_actors.config

@@ -1,8 +1,9 @@
 params {
     PARAMETER_PLAN = "FROM_ACTORS" 
     DECRYPTED_DATASET = "test/data/dataset.csv"
-    UKBB_BED_FILES = "test/data/bed/ukb_chr{1,2,3}.{bed,bim,fam}"
-    UKBB_BGEN_FILES = "test/data/bgen/ukb_chr{1,2,3}.{bgen,bgen.bgi,sample}"
+    COHORT = "UKBB"
+    BED_FILES = "test/data/bed/ukb_chr{1,2,3}.{bed,bim,fam}"
+    BGEN_FILES = "test/data/bgen/ukb_chr{1,2,3}.{bgen,bgen.bgi,sample}"
     QC_FILE = "test/data/qc_file.csv"
     LD_BLOCKS = "test/data/ld_blocks.txt"
     FLASHPCA_EXCLUSION_REGIONS = "test/data/exclusion_regions_hg19.txt"
@@ -25,4 +26,4 @@ process {
       memory = '10G'
       clusterOptions = { task.memory ? "-l mem_free=20G,h_vmem=${task.memory.bytes/task.cpus}" : null }
   }
-}
+}

conf/ci_jobs/from_param_files.config

@@ -2,8 +2,9 @@ params {
     PARAMETER_PLAN = "FROM_PARAM_FILES" 
     PARAMETER_FILES = "test/data/parameters/parameter_*" 
     DECRYPTED_DATASET = "test/data/dataset.csv"
-    UKBB_BED_FILES = "test/data/bed/ukb_chr{1,2,3}.{bed,bim,fam}"
-    UKBB_BGEN_FILES = "test/data/bgen/ukb_chr{1,2,3}.{bgen,bgen.bgi,sample}"
+    COHORT = "UKBB"
+    BED_FILES = "test/data/bed/ukb_chr{1,2,3}.{bed,bim,fam}"
+    BGEN_FILES = "test/data/bgen/ukb_chr{1,2,3}.{bgen,bgen.bgi,sample}"
     QC_FILE = "test/data/qc_file.csv"
     LD_BLOCKS = "test/data/ld_blocks.txt"
     FLASHPCA_EXCLUSION_REGIONS = "test/data/exclusion_regions_hg19.txt"
@@ -12,4 +13,4 @@ params {
     TRAITS_CONFIG = "test/data/ukbconfig_small.yaml"
     ESTIMATORFILE = "test/data/estimator.yaml"
     NB_VAR_ESTIMATORS = 0
-}
+}

main.nf

@@ -17,8 +17,10 @@ params.PVAL_SIEVE = 0.05
 
 params.OUTDIR = "$launchDir/results"
 
+params.COHORT = "UKBB"
 params.TRAITS_CONFIG = "NO_UKB_TRAIT_CONFIG"
 params.WITHDRAWAL_LIST = 'NO_WITHDRAWAL_LIST'
+params.QC_FILE = "NO_QC_FILE"
 
 params.PHENOTYPES_BATCH_SIZE = 0
 params.EXTRA_CONFOUNDERS = 'NO_EXTRA_CONFOUNDER'
@@ -61,7 +63,7 @@ workflow generateIIDGenotypes {
         qc_file = Channel.value(file("$params.QC_FILE"))
         flashpca_excl_reg = Channel.value(file("$params.FLASHPCA_EXCLUSION_REGIONS"))
         ld_blocks = Channel.value(file("$params.LD_BLOCKS"))
-        bed_files_ch = Channel.fromFilePairs("$params.UKBB_BED_FILES", size: 3, checkIfExists: true){ file -> file.baseName }
+        bed_files_ch = Channel.fromFilePairs("$params.BED_FILES", size: 3, checkIfExists: true){ file -> file.baseName }
 
         IIDGenotypes(flashpca_excl_reg, ld_blocks, bed_files_ch, qc_file, traits)
 
@@ -89,7 +91,7 @@ workflow generateTMLEEstimates {
 
     main:
         estimator_file = Channel.value(file("$params.ESTIMATORFILE", checkIfExists: true))
-        bgen_files = Channel.fromPath("$params.UKBB_BGEN_FILES", checkIfExists: true).collect()
+        bgen_files = Channel.fromPath("$params.BGEN_FILES", checkIfExists: true).collect()
 
         if (params.PARAMETER_PLAN == "FROM_ACTORS") {
             bqtls = Channel.value(file("$params.BQTLS"))
@@ -149,18 +151,23 @@ workflow generateSieveEstimates {
 }
 
 workflow {
-    // Extract traits
-    extractTraits()
+    // Extract traits for UKBB
+    if (params.COHORT == "UKBB") {
+        extractTraits()
+        phenoInput = extractTraits.out
+    } else {
+        phenoInput = Channel.fromPath("$params.DECRYPTED_DATASET", checkIfExists: true)
+    }
 
     // Generate IID Genotypes
-    generateIIDGenotypes(extractTraits.out)
+    generateIIDGenotypes(phenoInput)
 
     // Genetic confounders
     geneticConfounders(generateIIDGenotypes.out)
 
     // generate estimates
     generateTMLEEstimates(
-        extractTraits.out,
+        phenoInput,
         geneticConfounders.out,
     )
 
@@ -174,4 +181,4 @@ workflow {
     }
 
     MergeOutputs(generateTMLEEstimates.out.tmle_csvs.collect(), sieve_csv)
-}
+}

modules/genotypes.nf

@@ -1,6 +1,6 @@
 process filterBED{
     label 'bigmem'
-    container "olivierlabayle/tl-core:v0.3.0"
+    container "olivierlabayle/tl-core:brh_data_source"
     publishDir "$params.OUTDIR/qc_filtered_chromosomes", mode: 'symlink'
 
     input:
@@ -14,12 +14,23 @@ process filterBED{
 
     script:
         prefix = bedfiles[0].toString().minus('.bed')
-        """
-        TEMPD=\$(mktemp -d)
-        JULIA_DEPOT_PATH=\$TEMPD:/opt julia --project=/TargeneCore.jl --startup-file=no /TargeneCore.jl/bin/prepare_confounders.jl \
-        --input $prefix --output filtered.$prefix --qcfile $qcfile --maf-threshold $params.MAF_THRESHOLD --ld-blocks $ld_blocks --traits $traits filter
-        """
-
+
+        script="TEMPD=\$(mktemp -d)\n"
+        if (params.QC_FILE == "NO_QC_FILE") {
+            script += """
+                JULIA_DEPOT_PATH=\$TEMPD:/opt julia --project=/TargeneCore.jl --startup-file=no /TargeneCore.jl/bin/prepare_confounders.jl \
+                    --input $prefix --output filtered.$prefix --maf-threshold $params.MAF_THRESHOLD --ld-blocks $ld_blocks --traits $traits filter
+                """
+                .stripIndent()
+        } else {
+            script += """
+                JULIA_DEPOT_PATH=\$TEMPD:/opt julia --project=/TargeneCore.jl --startup-file=no /TargeneCore.jl/bin/prepare_confounders.jl \
+                    --input $prefix --output filtered.$prefix --qcfile $qcfile --maf-threshold $params.MAF_THRESHOLD --ld-blocks $ld_blocks --traits $traits filter
+                """
+                .stripIndent()
+        }
+
+        script
 }
 
 
@@ -95,4 +106,4 @@ workflow IIDGenotypes{
 
     emit:
         SampleQCFilter.out
-}
+}