Update annotation tests to use the latest data

oncokb · Nov 12, 2021 · 0214d87 · 0214d87
1 parent 70974f6
commit 0214d87
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 30 deletions.
diff --git a/...ub/workflows/compare-study-annotation.yml → .github/workflows/compare-annotation.yml b/...ub/workflows/compare-study-annotation.yml → .github/workflows/compare-annotation.yml
@@ -1,11 +1,14 @@
-# This workflow will install Python dependencies, run annotation against the master annotation for a particular study
+# This workflow will install Python dependencies, run annotation against the master annotation
 
-name: Compare Study Annotation
+name: Compare Annotation
 
 on:
   push:
     branches:
       - master
+  pull_request:
+    branches:
+      - master
 jobs:
   build:
     if: github.repository == 'oncokb/oncokb-annotator'
@@ -27,27 +30,25 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Annotate msk_impact_2017 study
+      - name: Annotate
         id: annotate
         env:
           ONCOKB_API_TOKEN: ${{ secrets.ONCOKB_BOT_API_TOKEN }}
           ONCOKB_OAUTH_TOKEN: ${{ secrets.ONCOKB_OAUTH_TOKEN }}
         run: |
           git checkout -b compare
 
-          STUDY=msk_impact_2017
-          DATAHUB_URL=https://media.githubusercontent.com/media/cBioPortal/datahub/42afc279efb8d9104aba36fa35bad3ec41921949/public/"$STUDY"
-
-          MUTATION_DATA_NAME=data_mutations_extended.txt
+          MUTATION_DATA_NAME=data_mutations_mskcc.txt
           CLINICAL_DATA_NAME=data_clinical_sample.txt
           FUSION_DATA_NAME=data_fusions.txt
           CNA_DATA_NAME=data_CNA.txt
 
           cd data || exit
-          curl -s "$DATAHUB_URL"/"$MUTATION_DATA_NAME" -O
-          curl -s "$DATAHUB_URL"/"$CLINICAL_DATA_NAME" -O
-          curl -s "$DATAHUB_URL"/"$FUSION_DATA_NAME" -O
-          curl -s "$DATAHUB_URL"/"$CNA_DATA_NAME" -O
+          curl -s -H "Authorization: token ${ONCOKB_OAUTH_TOKEN}" https://api.github.com/repos/knowledgesystems/oncokb-data/contents/annotation/annotator-test/data | jq -r '.[] | .download_url + " " + .name' | while IFS=' ' read -r downloadurl name; do
+            if [[ "$name" == "$FIEL_NAME_PREFIX"* ]]; then
+              curl -s "$downloadurl" -o $name
+            fi
+          done
           cd ..
 
           # create compare folder to add all annotated files
@@ -77,20 +78,17 @@ jobs:
           git add .
           git commit -m 'add analysis'
 
-          echo "::set-output name=STUDY::$STUDY"
-
       - name: Compare annotation result with the ones from master
         id: compare
         env:
-          STUDY: ${{steps.annotate.outputs.STUDY}}
           ONCOKB_OAUTH_TOKEN: ${{ secrets.ONCOKB_OAUTH_TOKEN }}
           FIEL_NAME_PREFIX: 'oncokb_data'
         run: |
           # remove everything under compare folder and replace wiht the ones from oncokb-data
           rm -f compare/*.txt
 
           cd compare || exit
-          curl -s -H "Authorization: token ${ONCOKB_OAUTH_TOKEN}" https://api.github.com/repos/knowledgesystems/oncokb-data/contents/annotation/$STUDY | jq -r '.[] | .download_url + " " + .name' | while IFS=' ' read -r downloadurl name; do
+          curl -s -H "Authorization: token ${ONCOKB_OAUTH_TOKEN}" https://api.github.com/repos/knowledgesystems/oncokb-data/contents/annotation/annotator-test/annotation | jq -r '.[] | .download_url + " " + .name' | while IFS=' ' read -r downloadurl name; do
             if [[ "$name" == "$FIEL_NAME_PREFIX"* ]]; then
               curl -s "$downloadurl" -o $name
             fi

diff --git a/...mpare-genomic-change-study-annotation.yml → ...ows/compare-genomic-change-annotation.yml b/...mpare-genomic-change-study-annotation.yml → ...ows/compare-genomic-change-annotation.yml
@@ -1,11 +1,14 @@
 # This workflow will install Python dependencies, run annotation against the master annotation for a particular study
 
-name: Compare Study Genomic Change Annotation
+name: Compare Genomic Change Annotation
 
 on:
   push:
     branches:
       - master
+  pull_request:
+    branches:
+      - master
 jobs:
   build:
     if: github.repository == 'oncokb/oncokb-annotator'
@@ -27,31 +30,29 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Annotate msk_impact_2017 study
+      - name: Annotate
         id: annotate
         env:
           ONCOKB_API_TOKEN: ${{ secrets.ONCOKB_BOT_API_TOKEN }}
           ONCOKB_OAUTH_TOKEN: ${{ secrets.ONCOKB_OAUTH_TOKEN }}
         run: |
           git checkout -b compare
 
-          STUDY=msk_impact_2017
-          DATAHUB_URL=https://media.githubusercontent.com/media/cBioPortal/datahub/42afc279efb8d9104aba36fa35bad3ec41921949/public/$STUDY
-
-          MUTATION_DATA_NAME=data_mutations_extended.txt
+          MUTATION_DATA_NAME=data_mutations_mskcc.txt
           CLINICAL_DATA_NAME=data_clinical_sample.txt
 
           cd data
-          curl -s $DATAHUB_URL/$MUTATION_DATA_NAME -O
-          curl -s $DATAHUB_URL/$CLINICAL_DATA_NAME -O
+          curl -s -H "Authorization: token ${ONCOKB_OAUTH_TOKEN}" https://api.github.com/repos/knowledgesystems/oncokb-data/contents/annotation/annotator-test/data | jq -r '.[] | .download_url + " " + .name' | while IFS=' ' read -r downloadurl name; do
+            if [[ "$name" == "$MUTATION_DATA_NAME" || "$name" == "$CLINICAL_DATA_NAME" ]]; then
+              curl -s "$downloadurl" -o $name
+            fi
+          done
           cd ..
 
           # create compare folder to add all annotated files
           mkdir compare
 
-          PREFIX=oncokb
-
-          OGCMAF="$PREFIX"_genomic_change_$MUTATION_DATA_NAME
+          OGCMAF=oncokb_genomic_change_$MUTATION_DATA_NAME
 
           python MafAnnotator.py -i data/$MUTATION_DATA_NAME -o compare/$OGCMAF -c data/$CLINICAL_DATA_NAME -b $ONCOKB_API_TOKEN -q Genomic_Change
 
@@ -61,21 +62,19 @@ jobs:
           git add .
           git commit -m 'add analysis'
 
-          echo "::set-output name=STUDY::$STUDY"
           echo "::set-output name=FILE_NAME::$OGCMAF"
 
       - name: Compare annotation result with the ones from master
         id: compare
         env:
-          STUDY: ${{steps.annotate.outputs.STUDY}}
           FILE_NAME: ${{steps.annotate.outputs.FILE_NAME}}
           ONCOKB_OAUTH_TOKEN: ${{ secrets.ONCOKB_OAUTH_TOKEN }}
         run: |
           # remove everything under compare folder and replace wiht the ones from oncokb-data
           rm -f compare/*.txt
 
           cd compare
-          curl -s -H "Authorization: token ${ONCOKB_OAUTH_TOKEN}" https://api.github.com/repos/knowledgesystems/oncokb-data/contents/annotation/$STUDY | jq -r '.[] | .download_url + " " + .name' | while IFS=' ' read -r downloadurl name; do
+          curl -s -H "Authorization: token ${ONCOKB_OAUTH_TOKEN}" https://api.github.com/repos/knowledgesystems/oncokb-data/contents/annotation/annotator-test/annotation | jq -r '.[] | .download_url + " " + .name' | while IFS=' ' read -r downloadurl name; do
             if [[ "$name" == "$FILE_NAME" ]]; then
               curl -s "$downloadurl" -o $name
             fi
@@ -90,4 +89,4 @@ jobs:
             git diff
             exit 1
           fi
-            
+