On Pull Request #113
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: On Pull Request | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| manual-pr-number: | |
| description: 'PR Number' | |
| required: true | |
| default: '10' | |
| #pull_request: | |
| #paths: | |
| # - '**/*.gpml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: false # to allow multiple runs to queue up rather than clobber | |
| jobs: | |
| get-pr: | |
| # This job determines the PR number and stores it as a GITHUB_OUTPUT environment variable | |
| # that can accessed by subsequent jobs. | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-number: ${{ steps.get-pr.outputs.pr-number }} | |
| steps: | |
| - name: Get PR | |
| id: get-pr | |
| # Get PR from triggering event, unless provided via manual workflow run | |
| run: | | |
| if [ -z "${{ inputs.manual-pr-number }}" ]; then | |
| echo "pr-number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "pr-number=${{ inputs.manual-pr-number }}" >> $GITHUB_OUTPUT | |
| fi | |
| get-gpml: | |
| # This job gets the GPML file and uploads it as an artifact | |
| # that can accessed by subsequent jobs. It also extracts key information from | |
| # the GPML file and starts a new PR description. | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| # | |
| # The branch name from the submitter's fork is also extracted and stored (if needed). | |
| needs: [get-pr] | |
| #if: ${{ needs.get-pr.outputs.pr-number }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| gpml-filepath: ${{ steps.get-gpml.outputs.gpml-filepath }} | |
| gpml-file: ${{ steps.get-gpml.outputs.gpml-file }} | |
| pr-desc: ${{ steps.get-gpml.outputs.pr-desc }} | |
| branch-name: ${{ steps.get-branch.outputs.branch-name }} | |
| env: | |
| PR_NUMBER: ${{ needs.get-pr.outputs.pr-number }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: refs/pull/${{ needs.get-pr.outputs.pr-number }}/head | |
| - name: Install XML processing tools | |
| # Used to parse XML values from GPML | |
| run: sudo apt-get install -y libxml2-utils | |
| - name: Get GPML | |
| id: get-gpml | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| GPML_FILEPATH=$(gh pr view $PR_NUMBER --json files --jq '.files.[].path' | grep '.gpml$') | |
| echo "Found GPML file at: $GPML_FILEPATH" | |
| echo "gpml-filepath=$GPML_FILEPATH" >> $GITHUB_OUTPUT #for subsequent jobs | |
| # Extract information from GPML | |
| GPML_FILE="$(basename ""$GPML_FILEPATH"")" | |
| echo "gpml-file=$GPML_FILE" >> $GITHUB_OUTPUT | |
| wpid="$($GPML_FILE | sed 's/.gpml//')" | |
| org="$(xmllint --xpath 'string(//*[local-name()="Pathway"]/@Organism)' "$GPML_FILEPATH")" | |
| name="$(xmllint --xpath 'string(//*[local-name()="Pathway"]/@Name)' "$GPML_FILEPATH")" | |
| desc="$(xmllint --xpath 'string(//*[local-name()="Comment" and @Source="WikiPathways-description"])' "$GPML_FILEPATH")" | |
| # Start PR description (newlines are important in NEW_DESCRIPTION string) | |
| NEW_DESCRIPTION=" | |
| ## Pathway Information | |
| **WPID**: $wpid | |
| **TITLE**: $name | |
| **ORGANISM**: $org | |
| **DESCRIPTION**: $desc | |
| --- | |
| " | |
| # Store PR description for compilation in final job | |
| # EOF base64 needed for storing multi-line strings as output variables | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| # Temporarily append PR description | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| Processing... | |
| " | |
| # Edit PR description | |
| gh pr edit $PR_NUMBER --body "$NEW_DESCRIPTION" | |
| - name: Upload GPML file as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gpml-file | |
| path: ${{ steps.get-gpml.outputs.gpml-filepath }} | |
| retention-days: 1 | |
| if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` | |
| - name: Get branch name | |
| id: get-branch | |
| # Determine PR branch from submitter's fork, if needed #TODO: prune if not needed | |
| run: | | |
| BRANCH_NAME=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| https://api.github.com/repos/${{ github.repository }}/pulls/${{needs.get-pr.outputs.pr-number}} \ | |
| | jq -r .head.ref) | |
| echo "branch-name=$BRANCH_NAME" | |
| echo "branch-name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| if ! git ls-remote --heads origin $BRANCH_NAME; then | |
| git push origin HEAD:refs/heads/$BRANCH_NAME | |
| fi | |
| metadata: | |
| # This job generates files dervied from the GPML, including info.json, datanodes.tsv and refs.tsv. | |
| # The meta-data-action Java application performs the bulk of the work: | |
| # https://github.com/wikipathways/meta-data-action | |
| # | |
| # The generated files are uploaded as artifacts to be downloaded and pushed together with other files | |
| # to another repo in a final sync job. This strategy allows for parallel job processing! | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| needs: [get-pr,get-gpml] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML artifact | |
| # Downloads gpml-file artifact to working dir | |
| # Note: Automatically unzips to provide GPML with original filename | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: gpml-file | |
| - name: Setup Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '11' | |
| - name: Cache meta-data-action with dependencies | |
| uses: actions/cache@v3 | |
| id: cacheMetaJar | |
| with: | |
| path: ./meta-data-action-1.1.2-jar-with-dependencies.jar | |
| key: cached-meta-data-action-${{ hashFiles('meta-data-action-1.1.2-jar-with-dependencies.jar') }} | |
| restore-keys: | | |
| cached-meta-data-action-${{ hashFiles('meta-data-action-1.1.2-jar-with-dependencies.jar') }} | |
| cached-meta-data-action- | |
| - name: Install deps | |
| run: | | |
| echo "Refreshing cached-meta-data-action" | |
| if [ ! -e ./meta-data-action-1.1.2-jar-with-dependencies.jar ]; then | |
| wget -O meta-data-action-1.1.2-jar-with-dependencies.jar https://github.com/wikipathways/meta-data-action/releases/download/v1.1.2/meta-data-action-1.1.2-jar-with-dependencies.jar | |
| fi | |
| - name: Cache and install dependencies | |
| uses: actions/cache@v3 | |
| id: cache | |
| with: | |
| path: ${{ github.workspace }}/Hs_Derby_Ensembl_108.bridge | |
| key: ${{ runner.os }}-java-Hs_Derby_Ensembl_108 | |
| restore-keys: | | |
| ${{ runner.os }}-java-Hs_Derby_Ensembl_108 | |
| ${{ runner.os }}-java-Hs_Derby_Ensembl_ | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| name: Install deps | |
| run: | | |
| cd "${{ github.workspace }}" | |
| if [ ! -e ./Hs_Derby_Ensembl_108.bridge ]; then | |
| wget -O Hs_Derby_Ensembl_108.bridge https://zenodo.org/record/7781913/files/Hs_Derby_Ensembl_108.bridge?download=1 | |
| fi | |
| - name: Generate gdb.config, fileNames.config, and fileDownloads.config | |
| run: scripts/meta-data-action/configGenerator.sh ./$GPML_FILE | |
| - name: Cache all bridge files | |
| uses: actions/cache@v3 | |
| id: cacheAllBridge | |
| with: | |
| path: | | |
| ./metabolites*.bridge | |
| ./Ag*.bridge | |
| ./An*.bridge | |
| ./At*.bridge | |
| ./Bs*.bridge | |
| ./Bt*.bridge | |
| ./Ce*.bridge | |
| ./Cf*.bridge | |
| ./Ci*.bridge | |
| ./Dr*.bridge | |
| ./Da*.bridge | |
| ./Dp*.bridge | |
| ./Dm*.bridge | |
| ./Ec*.bridge | |
| ./Gg*.bridge | |
| ./Fg*.bridge | |
| ./Gm*.bridge | |
| ./Hs*.bridge | |
| ./Hv*.bridge | |
| ./Ml*.bridge | |
| ./Mm*.bridge | |
| ./Mx*.bridge | |
| ./Oa*.bridge | |
| ./Ova*.bridge | |
| ./Oi*.bridge | |
| ./Oj*.bridge | |
| ./Pi*.bridge | |
| ./Pt*.bridge | |
| ./Qc*.bridge | |
| ./Rn*.bridge | |
| ./Sc*.bridge | |
| ./Sl*.bridge | |
| ./Ss*.bridge | |
| ./Vv*.bridge | |
| ./Xt*.bridge | |
| ./Zm*.bridge | |
| key: cached-bridge-files | |
| restore-keys: | | |
| cached-bridge-files | |
| - if: steps.cacheAllBridge.outputs.cache-hit != 'true' | |
| name: Install all bridge files | |
| run: | | |
| echo "Cache not found: cached-bridge-files" | |
| declare -a OrganismNames=("Metabolites" "Anopheles gambiae" "Aspergillus niger" "Arabidopsis thaliana" "Bacillus subtilis" "Bos taurus" "Caenorhabditis elegans" "Canis familiaris" "Ciona intestinalis" "Danio rerio" "Daphnia magna" "Daphnia pulex" "Drosophila melanogaster" "Escherichia coli" "Gallus gallus" "Fusarium graminearum" "Glycine max" "Homo sapiens" "Hordeum vulgare" "Macaca mulatta" "Mus musculus" "Mycobacterium tuberculosis" "Ornithorhynchus anatinus" "Ovis aries" "Oryza indica" "Oryza japonica" "Populus trichocarpa" "Pan troglodytes" "Equus caballus" "Rattus norvegicus" "Saccharomyces cerevisiae" "Solanum lycopersicum" "Sus scrofa" "Vitis vinifera" "Xenopus tropicalis" "Zea mays") | |
| for org in "${OrganismNames[@]}"; do | |
| echo "generating configuration files for "$org"" | |
| scripts/meta-data-action/configGenerator.sh "$org" | |
| echo "installing bridgedb files for "$org"" | |
| scripts/meta-data-action/installDependencies.sh "$org" | |
| done | |
| - name: Generate configs, install bridgeDb, generate info and tsv files | |
| run: | | |
| chmod 777 meta-data-action-1.1.2-jar-with-dependencies.jar | |
| f=./$GPML_FILE | |
| org="$(sed -n '/<Pathway /s/.*Organism=\(.*\)[^\n]*/\1/p' $f | tr -d '"' | tr -d '>' | tr -d '\r')" | |
| echo "generating configuration files for "$org"" | |
| scripts/meta-data-action/configGenerator.sh "$org" | |
| wpid="$(basename ""$f"" | sed 's/.gpml//')" | |
| cat gdb.config | |
| echo "generating info and datanode files for $wpid, organism "$org"" | |
| java -jar meta-data-action-1.1.2-jar-with-dependencies.jar local "$f" $(date --utc +%F) gdb.config "$org" | |
| # generated files are placed in "pathways" dir | |
| ls -R | |
| mkdir ./metadata | |
| mv pathways/"$wpid"-info.json metadata/. | |
| mv pathways/"$wpid"*.tsv metadata/. | |
| - name: Upload metadata files as artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: metadata | |
| path: metadata/ | |
| retention-days: 1 | |
| if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` | |
| - name: Report on meta-data-action | |
| id: report | |
| run: | | |
| # Verify generated files | |
| infojson=$(find . -path "./metadata/WP*info.json" -print -quit) | |
| dntsv=$(find . -path "./metadata/WP*datanodes.tsv" -print -quit) | |
| refstsv=$(find . -path "./metadata/WP*refs.tsv" -print -quit) | |
| # Update PR description | |
| NEW_DESCRIPTION=" | |
| ## Generate Metadata Files | |
| " | |
| if [[ -n $infojson ]]; then | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [x] info.json generated" | |
| else | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [] info.json generated" | |
| fi | |
| if [[ -n $dntsv ]]; then | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [x] datanodes.tsv generated" | |
| else | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [] datanodes.tsv generated" | |
| fi | |
| if [[ -n $refstsv ]]; then | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [x] refs.tsv generated" | |
| else | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [] refs.tsv generated" | |
| fi | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| pubmed: | |
| if: true | |
| needs: [get-pr,get-gpml, metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML artifact | |
| # Downloads gpml-file and metadata artifacts to working dir | |
| # Note: Automatically unzips to provide files with original filenames | |
| uses: actions/download-artifact@v3 | |
| - name: Run generate-references | |
| run: | | |
| echo "TODO: Run generate-references and attach output TSV to PR" | |
| - name: Report on pubmed | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION="- [x] references.tsv generated" | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| frontmatter: | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML artifact | |
| # Downloads gpml-file and metadata artifacts to working dir | |
| # Note: Automatically unzips to provide files with original filenames | |
| uses: actions/download-artifact@v3 | |
| - name: Run create_pathway_frontmatter.py | |
| run: | | |
| echo "TODO: Run create_pathway_frontmatter.py and attach output MD to PR" | |
| - name: Report on pubmed | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION="- [x] .md generated" | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| json-svg: | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML artifact | |
| # Downloads gpml-file and metadata artifacts to working dir | |
| # Note: Automatically unzips to provide files with original filenames | |
| uses: actions/download-artifact@v3 | |
| - name: Run generate-svgs | |
| run: | | |
| echo "TODO: Run generate-svgs and attach output JSON, SVG and PNG to PR" | |
| - name: Report on pubmed | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION="- [x] .json generated | |
| - [x] .svg generated | |
| - [x] .png generated | |
| *TODO: insert PNG here* | |
| " | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| testing: | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML artifact | |
| # Downloads gpml-file and metadata artifacts to working dir | |
| # Note: Automatically unzips to provide files with original filenames | |
| uses: actions/download-artifact@v3 | |
| - name: Perform automatic test | |
| run: | | |
| # TODO: perform testing | |
| - name: Report on pubmed | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION=' | |
| --- | |
| ## Automated Testing | |
| *As table* | |
| | # | Test name | Link | Notes | Result | | |
| |---|---|---|---|---| | |
| | 1 | Test one | | | \$\${\color{green}PASS}\$\$ | | |
| | 2 | Test two | | | \$\${\color{red}FAIL}\$\$ | | |
| | 3 | Test three | | | \$\${\color{green}PASS}\$\$ | | |
| *As checklist* | |
| - [ ] Interactions are connected | |
| - [ ] Datanodes are annotated with database references | |
| - [ ] Decription, consisting of 2-3 sentence overview of processes described in the pathway | |
| - [ ] At least one literature reference | |
| - [ ] At least one pathway ontology term | |
| - [ ] Pathway title conforms to the [guidelines](https://github.com/wikipathways/wikipathways-faq/discussions/24) | |
| --- | |
| ' | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| update-pr-desc: | |
| needs: [get-pr,get-gpml,metadata,pubmed,frontmatter,json-svg,testing] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: refs/pull/${{ needs.get-pr.outputs.pr-number }}/head | |
| - name: Compile and update PR description | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ needs.get-pr.outputs.pr-number }} | |
| run: | | |
| NEW_DESCRIPTION="${{ needs.get-gpml.outputs.pr-desc }} | |
| ${{ needs.metadata.outputs.pr-desc }} | |
| ${{ needs.pubmed.outputs.pr-desc }} | |
| ${{ needs.frontmatter.outputs.pr-desc }} | |
| ${{ needs.json-svg.outputs.pr-desc }} | |
| ${{ needs.testing.outputs.pr-desc }} | |
| " | |
| # Edit PR description | |
| gh pr edit $PR_NUMBER --body "$NEW_DESCRIPTION" |