feat: Upgrade mehari to 0.30.0

.github/workflows/build.yml

@@ -14,8 +14,8 @@ defaults:
     shell: bash -l {0}
 
 env:
-  MEHARI_VERSION: "0.26.1"
-  SNAKEMAKE_OUTPUT_CACHE: "/github/workspace/snakemake_cache"
+  MEHARI_VERSION: "0.30.0"
+  SNAKEMAKE_OUTPUT_CACHE: "${{ github.workspace }}/snakemake_cache"
 
 jobs:
   build_data_release:
@@ -35,6 +35,19 @@ jobs:
         id: date
         run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
 
+      - name: Setup apptainer
+        run: |
+          sudo add-apt-repository -y ppa:apptainer/ppa
+          sudo apt-get update
+          sudo apt install -y apptainer squashfuse
+
+      # See https://github.com/apptainer/apptainer/pull/2262
+      - name: Disable apparmor namespace restrictions for apptainer
+        run: |
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0 \
+                      -w kernel.apparmor_restrict_unprivileged_userns=0 \
+                      -w kernel.unprivileged_userns_clone=1
+
       - name: Cache data directory
         id: cache-mehari-data-tx
         uses: actions/cache@v4
@@ -47,22 +60,35 @@ jobs:
         uses: actions/cache@v4
         with:
           path: |
-            /github/workspace/snakemake_cache
             ~/snakemake_cache
             ${{ github.workspace }}/snakemake_cache
           key: ${{ matrix.genome_release }}-${{ matrix.source }}
 
-      - name: Run data build workflow
-        uses: snakemake/snakemake-github-action@v1
+      - name: Cache conda
+        uses: actions/cache@v3
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
         with:
-          directory: mehari-data-tx-workflow
-          snakefile: workflow/Snakefile
-          stagein: |
-            mkdir -p /github/workspace/snakemake_cache
-            mkdir -p ~/snakemake_cache
-            mkdir -p ${{ github.workspace }}/snakemake_cache
-          args: "--configfile config/config.yaml --sdm conda --show-failed-logs --cores 4 --jobs 4 results/${{ matrix.genome_release }}-${{ matrix.source }}/mehari/seqrepo/report/mehari_db_check.txt"
-          show-disk-usage-on-error: true
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yaml') }}
+
+      - name: Setup Miniconda and snakemake
+        uses: conda-incubator/setup-miniconda@v3.1.0
+        with:
+          auto-update-conda: true
+          conda-solver: "libmamba"
+          python-version: 3.12
+          environment-file: "environment.yaml"
+          conda-remove-defaults: "true"
+
+      - name: Run data build workflow
+        shell: bash -el {0}
+        run: |
+          mkdir -p ~/snakemake_cache
+          mkdir -p ${{ github.workspace }}/snakemake_cache
+          snakemake --verbose --configfile config/config.yaml --workflow-profile workflow/profiles/default --show-failed-logs --cores 4 --jobs 4 results/${{ matrix.genome_release }}-${{ matrix.source }}/mehari/seqrepo/report/mehari_db_check.txt
 
       - name: List files
         run: |

.github/workflows/release-please.yml

@@ -11,7 +11,7 @@ name: release-please
 #    shell: bash -l {0}
 
 env:
-  MEHARI_VERSION: "0.26.1"
+  MEHARI_VERSION: "0.30.0"
   SNAKEMAKE_OUTPUT_CACHE: "/github/workspace/snakemake_cache"
 
 jobs:
@@ -53,6 +53,19 @@ jobs:
         id: date
         run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
 
+      - name: Setup apptainer
+        run: |
+          sudo add-apt-repository -y ppa:apptainer/ppa
+          sudo apt-get update
+          sudo apt install -y apptainer squashfuse
+
+      # See https://github.com/apptainer/apptainer/pull/2262
+      - name: Disable apparmor namespace restrictions for apptainer
+        run: |
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0 \
+                      -w kernel.apparmor_restrict_unprivileged_userns=0 \
+                      -w kernel.unprivileged_userns_clone=1
+
       - name: Cache data directory
         id: cache-mehari-data-tx
         uses: actions/cache@v4
@@ -65,23 +78,36 @@ jobs:
         uses: actions/cache@v4
         with:
           path: |
-            /github/workspace/snakemake_cache
             ~/snakemake_cache
             ${{ github.workspace }}/snakemake_cache
           key: ${{ matrix.genome_release }}-${{ matrix.source }}
 
-      - name: Run data build workflow
-        if: ${{ needs.release-please.outputs.release_created }}
-        uses: snakemake/snakemake-github-action@v1
+      - name: Cache conda
+        uses: actions/cache@v3
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
         with:
-          directory: mehari-data-tx-workflow
-          snakefile: workflow/Snakefile
-          stagein: |
-            mkdir -p /github/workspace/snakemake_cache
-            mkdir -p ~/snakemake_cache
-            mkdir -p ${{ github.workspace }}/snakemake_cache
-          args: "--configfile config/config.yaml --sdm conda --show-failed-logs --cores 4 --jobs 4 results/${{ matrix.genome_release }}-${{ matrix.source }}/mehari/seqrepo/report/mehari_db_check.txt"
-          show-disk-usage-on-error: true
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yaml') }}
+
+      - name: Setup Miniconda and snakemake
+        uses: conda-incubator/setup-miniconda@v3.1.0
+        with:
+          auto-update-conda: true
+          conda-solver: "libmamba"
+          python-version: 3.12
+          environment-file: "environment.yaml"
+          conda-remove-defaults: "true"
+
+      - name: Run data build workflow
+        shell: bash -el {0}
+        run: |
+          mkdir -p ~/snakemake_cache
+          mkdir -p ${{ github.workspace }}/snakemake_cache
+          snakemake --verbose --configfile config/config.yaml --workflow-profile workflow/profiles/default --show-failed-logs --cores 4 --jobs 4 results/${{ matrix.genome_release }}-${{ matrix.source }}/mehari/seqrepo/report/mehari_db_check.txt
+
 
       - name: upload release assets
         if: ${{ needs.release-please.outputs.release_created }}

config/config.yaml

@@ -248,4 +248,5 @@ human-phenotype-ontology:
 
 # mehari version to use (currently ignored)
 mehari:
-  version: 0.26.6
+  version: 0.30.0
+  docker: "docker://ghcr.io/varfish-org/mehari:pr-603"

environment.yaml

@@ -0,0 +1,5 @@
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - snakemake =8.25.3

workflow/Snakefile

@@ -1,6 +1,6 @@
 from snakemake.utils import validate, min_version
 
-min_version("8.16.0")
+min_version("8.25.3")
 
 
 configfile: "config/config.yaml"

workflow/envs/mehari.yaml

@@ -3,4 +3,4 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - mehari ==0.26.1
+  - mehari ==0.30.0

workflow/profiles/default/config.v8+.yaml

@@ -1,5 +1,7 @@
-software-deployment-method: conda
+software-deployment-method:
+  - conda
+  - apptainer
 cache: True
+apptainer-args: "-B ${SNAKEMAKE_OUTPUT_CACHE}:${SNAKEMAKE_OUTPUT_CACHE}:rw"
 resources:
   ratelimit: 3
-

workflow/rules/common.smk

@@ -7,10 +7,31 @@ def get_alias(wildcards: Wildcards) -> str:
     return f"{wildcards.assembly}-{wildcards.source}"
 
 
-def genome_release(wildcards: Wildcards) -> str:
+def genome_assembly(wildcards: Wildcards) -> str:
     return wildcards.assembly.lower()
 
 
+def genome_assembly_version_parameter(_wildcards: Wildcards) -> str:
+    # TODO determine e.g. patch version for GRCh37/38
+    return ""
+
+
+def transcript_source(wildcards: Wildcards) -> str:
+    return wildcards.source.lower()
+
+
+def transcript_source_version_parameter(wildcards: Wildcards) -> str:
+    if wildcards.source.lower() == "ensembl":
+        release = config["reference"][wildcards.assembly]["ensembl"]["release"]
+        return f"--transcript-source-version {release}"
+    else:
+        return ""
+
+
+def cdot_version(wildcards: Wildcards) -> str:
+    return config["sources"][get_alias(wildcards)]["cdot"]["release"]
+
+
 def get_ensembl_sequence_param(param: str) -> Callable[[Wildcards], str]:
     def inner(wildcards):
         return config["reference"][wildcards.assembly]["ensembl"][param]
@@ -142,3 +163,7 @@ def get_genes_to_disease_download_url() -> str:
 def get_known_issues(wildcards: Wildcards) -> list[str]:
     alias = get_alias(wildcards)
     return config["sources"][alias].get("known_issues", [])
+
+
+def get_mehari_docker_url() -> str:
+    return config["mehari"]["docker"]

workflow/rules/mehari.smk

@@ -15,20 +15,28 @@ rule mehari_build_txs_db:
             else ""
         ),
         cdot=get_mehari_cdot_param_string,
-        genome_release=genome_release,
+        assembly=genome_assembly,
+        assembly_version=genome_assembly_version_parameter,
+        cdot_version=cdot_version,
+        transcript_source=transcript_source,
+        transcript_source_version=transcript_source_version_parameter,
     log:
         "logs/{assembly}-{source}/mehari/seqrepo/build_txs_db.log",
     benchmark:
         "benchmarks/{assembly}-{source}/mehari/seqrepo/build_txs_db.tsv"
-    conda:
-        "../envs/mehari.yaml"
+    container:
+        get_mehari_docker_url()
     shell:
         """
         mehari db create \
         --path-out {output.txs} \
         --path-seqrepo-instance {input.seqrepo_instance} \
         {params.cdot} \
-        --genome-release {params.genome_release} \
+        --cdot-version {params.cdot_version} \
+        --assembly {params.assembly} \
+        {params.assembly_version} \
+        --transcript-source {params.transcript_source} \
+        {params.transcript_source_version} \
         --threads {threads} \
         {params.mane} 2> {log}
         """

workflow/rules/validate.smk

@@ -28,12 +28,11 @@ rule check_mehari_db:
             category="{assembly}-{source}",
         ),
     params:
-        known_issues=get_known_issues,
         cdot=get_mehari_check_cdot_param_string,
     log:
         "logs/{assembly}-{source}/mehari/seqrepo/check.log",
-    conda:
-        "../envs/mehari.yaml"
+    container:
+        get_mehari_docker_url()
     shell:
         """(
         mehari db check \

workflow/schemas/config.schema.yaml

@@ -158,8 +158,11 @@ properties:
     properties:
       version:
         type: string
+      docker:
+        type: string
     required:
       - version
+      - docker
 required:
   - reference
   - sources