From a3b42aca2dd23229829f48299ccb62af899b76f1 Mon Sep 17 00:00:00 2001
From: verku <verena.kutschera@scilifelab.se>
Date: Tue, 26 Nov 2024 15:47:26 +0100
Subject: [PATCH] Simplify compute resource specification

---
 .../slurm/profile/config_plugin_dardel.yaml   | 547 +++++-------------
 1 file changed, 158 insertions(+), 389 deletions(-)

diff --git a/config/slurm/profile/config_plugin_dardel.yaml b/config/slurm/profile/config_plugin_dardel.yaml
index 3d79988..4d70207 100644
--- a/config/slurm/profile/config_plugin_dardel.yaml
+++ b/config/slurm/profile/config_plugin_dardel.yaml
@@ -12,400 +12,169 @@ apptainer-args: "-B /cfs/klemming"
 default-resources: 
   slurm_account: XXX-XX-XXX # update this to your slurm account
   slurm_partition: shared # use Dardel’s shared partition
-  runtime: 120 # default runtime in minutes
-  mem_mb: 6400
   nodes: 1 # one node on Dardel from the shared partition
   ntasks: 1 # number of concurrent tasks / ranks
-  cpus_per_task: 8 # number of hyperthreads per task, corresponds to 0.8 GB RAM
+  threads: 10 # number of hyperthreads per task, corresponds to 0.8 GB RAM
+  mem_mb: f"{threads*800}" # calculate available memory in MB for specific number of threads
+  runtime: 120 # default runtime in minutes
 
 ## set-threads: map rule names to threads
 set-threads:
-  repeatmodeler: 64
-  repeatmasker: 128
-  historical_fastq_before_group: 16
-  fastqc_historical_raw: 16
-  fastqc_modern_raw: 16
-  fastp_historical: 32
-  fastp_modern: 32
-  fastqc_historical_merged: 16
-  fastqc_historical_unmerged: 16
-  fastqc_modern_trimmed: 16
-  map_historical: 64
-  sai2bam: 64
-  map_modern: 64
-  sorted_bam_qualimap: 64
-  merge_historical_bams_per_index: 16
-  merge_modern_bams_per_index: 16
-  merged_index_bam_qualimap: 64
-  rmdup_historical_bams: 48
-  rmdup_modern_bams: 16
-  rmdup_bam_qualimap: 64
-  merge_historical_bams_per_sample: 16
-  merge_modern_bams_per_sample: 16
-  merged_sample_bam_qualimap: 64
-  indel_realigner_targets: 64
-  indel_realigner: 64
-  realigned_bam_qualimap: 64
-  realigned_bam_group: 16
-  rescale_historical: 32
-  rescaled_bam_qualimap: 64
-  rescaled_bam_group: 16
-  subsample_bams: 16
-  subsampled_bam_qualimap: 48
-  variant_calling: 32
-  sort_vcfs: 16
-  CpG_genotype_bed_formatting_group: 32
-  all_CpG_bed_formatting_group: 16
-  make_noCpG_bed: 16
-  CpG_repeats_bed_formatting_group: 16
-  merge_noCpG_noRepeats_beds: 16
-  make_noCpG_repma_bed: 16
-  intersect_sexchr_repma_beds: 16
-  intersect_autos_repma_beds: 16
-  intersect_sexchr_CpG_repma_beds: 16
-  intersect_autos_CpG_repma_beds: 16
-  remove_CpG_vcf: 48
-  CpG_vcf2bcf: 16
-  remove_snps_near_indels: 16
-  filter_vcfs_qual_dp: 16
-  filter_vcfs_allelic_balance: 16
-  remove_repeats_vcf: 48
-  filtered_vcf2bcf: 16
-  merge_all_vcfs: 48
-  filter_vcf_biallelic: 16
-  filter_vcf_missing: 16
-  remove_chromosomes: 16
-  repmasked_bcf2vcf: 16
-  filter_biallelic_missing_vcf: 48
-  vcf2plink_pca: 16
-  plink_eigenvec: 8
-  filter_vcf_hwe: 16
-  vcf2plink_hwe: 16
-  outgroup_fastqc: 16
-  align2target: 64
-  bam2fasta: 16
-  concatenate_fasta_per_contig: 16
-  compute_gerp: 32
-  gerp2coords: 16
-  get_ancestral_state: 16
-  produce_contig_out: 16
-  merge_per_chunk: 16
-  merge_gerp_gz: 16
-  gerp_derived_alleles: 16
-  merge_gerp_alleles_per_chunk: 32
-  merge_gerp_alleles_gz: 32
-  relative_mutational_load_per_sample: 16
+  repeatmodeler: 80
+  repeatmasker: 80
+  historical_fastq_before_group: 20
+  fastqc_historical_raw: 20
+  fastqc_modern_raw: 20
+  fastp_historical: 40
+  fastp_modern: 40
+  fastqc_historical_merged: 20
+  fastqc_historical_unmerged: 20
+  fastqc_modern_trimmed: 20
+  map_historical: 80
+  sai2bam: 80
+  map_modern: 80
+  sorted_bam_qualimap: 80
+  merge_historical_bams_per_index: 20
+  merge_modern_bams_per_index: 20
+  merged_index_bam_qualimap: 80
+  rmdup_historical_bams: 60
+  rmdup_modern_bams: 20
+  rmdup_bam_qualimap: 80
+  merge_historical_bams_per_sample: 20
+  merge_modern_bams_per_sample: 20
+  merged_sample_bam_qualimap: 80
+  indel_realigner_targets: 80
+  indel_realigner: 80
+  realigned_bam_qualimap: 80
+  realigned_bam_group: 20
+  rescale_historical: 40
+  rescaled_bam_qualimap: 80
+  rescaled_bam_group: 20
+  subsample_bams: 20
+  subsampled_bam_qualimap: 60
+  variant_calling: 40
+  sort_vcfs: 20
+  CpG_genotype_bed_formatting_group: 40
+  all_CpG_bed_formatting_group: 20
+  make_noCpG_bed: 20
+  CpG_repeats_bed_formatting_group: 20
+  merge_noCpG_noRepeats_beds: 20
+  make_noCpG_repma_bed: 20
+  intersect_sexchr_repma_beds: 20
+  intersect_autos_repma_beds: 20
+  intersect_sexchr_CpG_repma_beds: 20
+  intersect_autos_CpG_repma_beds: 20
+  remove_CpG_vcf: 60
+  CpG_vcf2bcf: 20
+  remove_snps_near_indels: 20
+  filter_vcfs_qual_dp: 20
+  filter_vcfs_allelic_balance: 20
+  remove_repeats_vcf: 60
+  filtered_vcf2bcf: 20
+  merge_all_vcfs: 60
+  filter_vcf_biallelic: 20
+  filter_vcf_missing: 20
+  remove_chromosomes: 20
+  repmasked_bcf2vcf: 20
+  filter_biallelic_missing_vcf: 60
+  vcf2plink_pca: 20
+  filter_vcf_hwe: 20
+  vcf2plink_hwe: 20
+  outgroup_fastqc: 20
+  align2target: 80
+  bam2fasta: 20
+  concatenate_fasta_per_contig: 20
+  compute_gerp: 40
+  gerp2coords: 20
+  get_ancestral_state: 20
+  produce_contig_out: 20
+  merge_per_chunk: 20
+  merge_gerp_gz: 20
+  gerp_derived_alleles: 20
+  merge_gerp_alleles_per_chunk: 40
+  merge_gerp_alleles_gz: 40
+  relative_mutational_load_per_sample: 20
 
 ## set-resources: map rule names to resources in general
 set-resources:
-  repeatmodeler:
-    runtime: 10080
-    mem_mb: 102400
-    cpus_per_task: 64
-  repeatmasker:
-    runtime: 10080
-    mem_mb: 102400
-    cpus_per_task: 128
-  historical_fastq_before_group:
-    runtime: 600
-    mem_mb: 12800
-    cpus_per_task: 16
-  fastqc_historical_raw:
-    mem_mb: 12800
-    cpus_per_task: 16
-  fastqc_modern_raw:
-    mem_mb: 12800
-    cpus_per_task: 16
-  fastp_historical:
-    runtime: 600
-    mem_mb: 25600
-    cpus_per_task: 32
-  fastp_modern:
-    runtime: 600
-    mem_mb: 25600
-    cpus_per_task: 32
-  fastqc_historical_merged:
-    mem_mb: 12800
-    cpus_per_task: 16
-  fastqc_historical_unmerged:
-    mem_mb: 12800
-    cpus_per_task: 16
-  fastqc_modern_trimmed:
-    mem_mb: 12800
-    cpus_per_task: 16
-  map_historical:
-    runtime: 10080
-    mem_mb: 51200
-    cpus_per_task: 64
-  sai2bam:
-    runtime: 10080
-    mem_mb: 51200
-    cpus_per_task: 64
-  map_modern:
-    runtime: 10080
-    mem_mb: 51200
-    cpus_per_task: 64
-  sorted_bam_qualimap:
-    runtime: 5760
-    mem_mb: 51200
-    cpus_per_task: 64
-  merge_historical_bams_per_index:
-    runtime: 2880
-    mem_mb: 12800
-    cpus_per_task: 16
-  merge_modern_bams_per_index:
-    runtime: 2880
-    mem_mb: 12800
-    cpus_per_task: 16
-  merged_index_bam_qualimap:
-    runtime: 5760
-    mem_mb: 51200
-    cpus_per_task: 64
-  rmdup_historical_bams:
-    runtime: 4320
-    mem_mb: 38400
-    cpus_per_task: 48
-  rmdup_modern_bams:
-    runtime: 4320
-    mem_mb: 12800
-    cpus_per_task: 16
-  rmdup_bam_qualimap:
-    runtime: 5760
-    mem_mb: 51200
-    cpus_per_task: 64
-  merge_historical_bams_per_sample:
-    runtime: 2880
-    mem_mb: 12800
-    cpus_per_task: 16
-  merge_modern_bams_per_sample:
-    runtime: 2880
-    mem_mb: 12800
-    cpus_per_task: 16
-  merged_sample_bam_qualimap:
-    runtime: 5760
-    mem_mb: 51200
-    cpus_per_task: 64
-  indel_realigner_targets:
-    runtime: 7200
-    mem_mb: 51200
-    cpus_per_task: 64
-  indel_realigner:
-    runtime: 7200
-    mem_mb: 51200
-    cpus_per_task: 64
-  realigned_bam_qualimap:
-    runtime: 5760
-    mem_mb: 51200
-    cpus_per_task: 64
-  realigned_bam_group:
-    runtime: 2880
-    mem_mb: 12800
-    cpus_per_task: 16
-  rescale_historical:
-    runtime: 4320
-    mem_mb: 25600
-    cpus_per_task: 32
-  rescaled_bam_qualimap:
-    runtime: 5760
-  rescaled_bam_group:
-    runtime: 600
-  filter_bam_mapped_mq:
-    runtime: 1440
-  subsample_bams:
-    runtime: 5760
-    mem_mb: 12800
-    cpus_per_task: 16
-  subsampled_bam_qualimap:
-    runtime: 5760
-    mem_mb: 38400
-    cpus_per_task: 48
-  subsampled_bam_group:
-    runtime: 2880
-  variant_calling:
-    runtime: 2880
-    mem_mb: 25600
-    cpus_per_task: 32
-  sort_vcfs:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  sorted_bcf2vcf:
-    runtime: 300
-  make_CpG_genotype_bed:
-    runtime: 1440
-  CpG_genotype_bed_formatting_group:
-    runtime: 1440
-    mem_mb: 25600
-    cpus_per_task: 32
-  all_CpG_bed_formatting_group:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  make_noCpG_bed:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  CpG_repeats_bed_formatting_group:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  merge_noCpG_noRepeats_beds:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  make_noCpG_repma_bed:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  make_autosomes_bed:
-    runtime: 300
-  intersect_sexchr_repma_beds:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  intersect_autos_repma_beds:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  intersect_sexchr_CpG_repma_beds:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  intersect_autos_CpG_repma_beds:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  bam2pro_autos:
-    runtime: 1440
-  bam2pro_sexchr:
-    runtime: 1440
-  bam2pro_all:
-    runtime: 1440
-  remove_CpG_vcf:
-    runtime: 300
-    mem_mb: 38400
-    cpus_per_task: 48
-  CpG_vcf2bcf:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  remove_snps_near_indels:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  filter_vcfs_qual_dp:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  filter_vcfs_allelic_balance:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  remove_repeats_vcf:
-    runtime: 300
-    mem_mb: 38400
-    cpus_per_task: 48
-  filtered_vcf2bcf:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  merge_all_vcfs:
-    runtime: 4320
-    mem_mb: 38400
-    cpus_per_task: 48
-  filter_vcf_biallelic:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  filter_vcf_missing:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  remove_chromosomes:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  extract_historical_samples:
-    runtime: 300
-  extract_modern_samples:
-    runtime: 300
-  repmasked_bcf2vcf:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  filter_biallelic_missing_vcf:
-    runtime: 1440
-    mem_mb: 38400
-    cpus_per_task: 48
-  vcf2plink_pca:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  filter_vcf_hwe:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  vcf2plink_hwe:
-    runtime: 300
-    mem_mb: 12800
-    cpus_per_task: 16
-  build_snpEff_db:
-    runtime: 300
-  annotate_vcf:
-    runtime: 300
-  outgroup_fastqc:
-    runtime: 16
-    mem_mb: 12800
-    cpus_per_task: 16
-  align2target:
-    runtime: 4320
-    mem_mb: 51200
-    cpus_per_task: 64
-  bam2fasta:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  concatenate_fasta_per_contig:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  compute_gerp:
-    runtime: 1440
-    mem_mb: 25600
-    cpus_per_task: 32
-  gerp2coords:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  get_ancestral_state:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  produce_contig_out:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  merge_per_chunk:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  merge_gerp_gz:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
-  gerp_derived_alleles:
-    runtime: 10080
-    mem_mb: 12800
-    cpus_per_task: 16
-  merge_gerp_alleles_per_chunk:
-    runtime: 1440
-    mem_mb: 25600
-    cpus_per_task: 32
-  merge_gerp_alleles_gz:
-    runtime: 1440
-    mem_mb: 25600
-    cpus_per_task: 32
-  relative_mutational_load_per_sample:
-    runtime: 1440
-    mem_mb: 12800
-    cpus_per_task: 16
+  repeatmodeler.runtime: 10080
+  repeatmasker.runtime: 10080
+  historical_fastq_before_group.runtime: 600
+  fastp_historical.runtime: 600
+  fastp_modern.runtime: 600
+  map_historical.runtime: 10080
+  sai2bam.runtime: 10080
+  map_modern.runtime: 10080
+  sorted_bam_qualimap.runtime: 5760
+  merge_historical_bams_per_index.runtime: 2880
+  merge_modern_bams_per_index.runtime: 2880
+  merged_index_bam_qualimap.runtime: 5760
+  rmdup_historical_bams.runtime: 4400
+  rmdup_modern_bams.runtime: 4400
+  rmdup_bam_qualimap.runtime: 5760
+  merge_historical_bams_per_sample.runtime: 2880
+  merge_modern_bams_per_sample.runtime: 2880
+  merged_sample_bam_qualimap.runtime: 5760
+  indel_realigner_targets.runtime: 7200
+  indel_realigner.runtime: 7200
+  realigned_bam_qualimap.runtime: 5760
+  realigned_bam_group.runtime: 2880
+  rescale_historical.runtime: 4400
+  rescaled_bam_qualimap.runtime: 5760
+  rescaled_bam_group.runtime: 600
+  filter_bam_mapped_mq.runtime: 1440
+  subsample_bams.runtime: 5760
+  subsampled_bam_qualimap.runtime: 5760
+  subsampled_bam_group.runtime: 2880
+  variant_calling.runtime: 2880
+  sort_vcfs.runtime: 1440
+  sorted_bcf2vcf.runtime: 300
+  make_CpG_genotype_bed.runtime: 1440
+  CpG_genotype_bed_formatting_group.runtime: 1440
+  all_CpG_bed_formatting_group.runtime: 1440
+  make_noCpG_bed.runtime: 300
+  CpG_repeats_bed_formatting_group.runtime: 1440
+  merge_noCpG_noRepeats_beds.runtime: 300
+  make_noCpG_repma_bed.runtime: 300
+  make_autosomes_bed.runtime: 300
+  intersect_sexchr_repma_beds.runtime: 300
+  intersect_autos_repma_beds.runtime: 300
+  intersect_sexchr_CpG_repma_beds.runtime: 300
+  intersect_autos_CpG_repma_beds.runtime: 300
+  bam2pro_autos.runtime: 1440
+  bam2pro_sexchr.runtime: 1440
+  bam2pro_all.runtime: 1440
+  remove_CpG_vcf.runtime: 300
+  CpG_vcf2bcf.runtime: 300
+  remove_snps_near_indels.runtime: 300
+  filter_vcfs_qual_dp.runtime: 300
+  filter_vcfs_allelic_balance.runtime: 300
+  remove_repeats_vcf.runtime: 300
+  filtered_vcf2bcf.runtime: 300
+  merge_all_vcfs.runtime: 4400
+  filter_vcf_biallelic.runtime: 1440
+  filter_vcf_missing.runtime: 1440
+  remove_chromosomes.runtime: 1440
+  extract_historical_samples.runtime: 300
+  extract_modern_samples.runtime: 300
+  repmasked_bcf2vcf.runtime: 300
+  filter_biallelic_missing_vcf.runtime: 1440
+  vcf2plink_pca.runtime: 300
+  filter_vcf_hwe.runtime: 300
+  vcf2plink_hwe.runtime: 300
+  build_snpEff_db.runtime: 300
+  annotate_vcf.runtime: 300
+  outgroup_fastqc.runtime: 20
+  align2target.runtime: 4400
+  bam2fasta.runtime: 1440
+  concatenate_fasta_per_contig.runtime: 1440
+  compute_gerp.runtime: 1440
+  gerp2coords.runtime: 1440
+  get_ancestral_state.runtime: 1440
+  produce_contig_out.runtime: 1440
+  merge_per_chunk.runtime: 1440
+  merge_gerp_gz.runtime: 1440
+  gerp_derived_alleles.runtime: 10080
+  merge_gerp_alleles_per_chunk.runtime: 1440
+  merge_gerp_alleles_gz.runtime: 1440
+  relative_mutational_load_per_sample.runtime: 1440