On Pull Request #140
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: On Pull Request | |
| # This is the primary workflow to process pathway edits coming in as GPML files from | |
| # PathVisio. A new pull request should be create per pathway. Subsequent edits to the | |
| # same GPML can accumulate in the same pull request while in review. | |
| # | |
| # The outputs of this workflow will be temporarily stored as PR artifacts to allow access | |
| # by subsequent jobs and to allow parallelization. At the end of the worflow, the outputs | |
| # will be pushed to the main website (wikipathways.github.io repo) for transparent review. | |
| # | |
| # The acceptance/merge of a processed pull request will trigger a secondary workflow to | |
| # migrate output files to published folders in main website repo, as well as the assets | |
| # and homology repos. | |
| # | |
| # Inputs: Each pull request should have one and only one GPML file attached. | |
| # | |
| # Outputs: | |
| # - info.json | |
| # - datanodes.tsv | |
| # - bibliography.tsv (via refs.tsv) | |
| # - .json .svg and .png files | |
| # - .md file | |
| # - TODO: test results? | |
| # | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| manual-pr-number: | |
| description: 'PR Number' | |
| required: true | |
| default: '10' | |
| #pull_request: | |
| #paths: | |
| # - '**/*.gpml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: false # to allow multiple runs to queue up rather than clobber | |
| jobs: | |
| get-pr: | |
| # This job determines the PR number and stores it as a GITHUB_OUTPUT environment variable | |
| # that can accessed by subsequent jobs. | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-number: ${{ steps.get-pr.outputs.pr-number }} | |
| steps: | |
| - name: Get PR | |
| id: get-pr | |
| # Get PR from triggering event, unless provided via manual workflow run | |
| run: | | |
| if [ -z "${{ inputs.manual-pr-number }}" ]; then | |
| echo "pr-number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "pr-number=${{ inputs.manual-pr-number }}" >> $GITHUB_OUTPUT | |
| fi | |
| get-gpml: | |
| # This job gets the GPML file, renames it (if new), and uploads it as an artifact | |
| # that can accessed by subsequent jobs. The GPML filename is also stored as a | |
| # GITHUB_OUTPUT environment variable. The job also extracts key information from | |
| # the GPML file and starts a new PR description to let reviewers know the processing status. | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| # | |
| # The branch name from the submitter's fork is also extracted and stored (if needed). | |
| needs: [get-pr] | |
| #if: ${{ needs.get-pr.outputs.pr-number }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| gpml-filepath: ${{ steps.get-gpml.outputs.gpml-filepath }} | |
| gpml-file: ${{ steps.get-gpml.outputs.gpml-file }} | |
| pr-desc: ${{ steps.get-gpml.outputs.pr-desc }} | |
| branch-name: ${{ steps.get-branch.outputs.branch-name }} | |
| env: | |
| PR_NUMBER: ${{ needs.get-pr.outputs.pr-number }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: refs/pull/${{ needs.get-pr.outputs.pr-number }}/head | |
| - name: Install XML processing tools | |
| # Used to parse XML values from GPML | |
| run: sudo apt-get install -y libxml2-utils | |
| - name: Get GPML | |
| id: get-gpml | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| GPML_FILEPATH=$(gh pr view $PR_NUMBER --json files --jq '.files.[].path' | grep '.gpml$') | |
| echo "Found GPML file at: $GPML_FILEPATH" | |
| GPML_FILE="$(basename ""$GPML_FILEPATH"")" | |
| # Rename if new pathway | |
| first_digit=${GPML_FILE:2:1} | |
| if [ "$first_digit" == "0" ]; then | |
| echo "Temporarily renaming new pathway per PR number." | |
| GPML_FILE="WP0${PR_NUMBER}.gpml" | |
| fi | |
| cp "$GPML_FILEPATH" ./"$GPML_FILE" | |
| GPML_FILEPATH="./${GPML_FILE}" | |
| echo "gpml-file=$GPML_FILE" >> $GITHUB_OUTPUT | |
| echo "gpml-filepath=$GPML_FILEPATH" >> $GITHUB_OUTPUT #for subsequent jobs | |
| # Extract information from GPML | |
| wpid="$(echo $GPML_FILE | sed 's/.gpml//')" | |
| org="$(xmllint --xpath 'string(//*[local-name()="Pathway"]/@Organism)' "$GPML_FILEPATH")" | |
| name="$(xmllint --xpath 'string(//*[local-name()="Pathway"]/@Name)' "$GPML_FILEPATH")" | |
| desc="$(xmllint --xpath 'string(//*[local-name()="Comment" and @Source="WikiPathways-description"])' "$GPML_FILEPATH")" | |
| # Start PR description (newlines are important in NEW_DESCRIPTION string) | |
| NEW_DESCRIPTION=" | |
| ## Pathway Information | |
| **WPID**: $wpid | |
| **TITLE**: $name | |
| **ORGANISM**: $org | |
| **DESCRIPTION**: $desc | |
| --- | |
| " | |
| # Store PR description for compilation in final job | |
| # EOF base64 needed for storing multi-line strings as output variables | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| # Temporarily append PR description | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| Processing... | |
| " | |
| # Edit PR description | |
| gh pr edit $PR_NUMBER --body "$NEW_DESCRIPTION" | |
| - name: Upload GPML file as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gpml-file | |
| path: ${{ steps.get-gpml.outputs.gpml-filepath }} | |
| retention-days: 1 | |
| if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` | |
| - name: Get branch name | |
| id: get-branch | |
| # Determine PR branch from submitter's fork, if needed #TODO: prune if not needed | |
| run: | | |
| BRANCH_NAME=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| https://api.github.com/repos/${{ github.repository }}/pulls/${{needs.get-pr.outputs.pr-number}} \ | |
| | jq -r .head.ref) | |
| echo "branch-name=$BRANCH_NAME" | |
| echo "branch-name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| if ! git ls-remote --heads origin $BRANCH_NAME; then | |
| git push origin HEAD:refs/heads/$BRANCH_NAME | |
| fi | |
| metadata: | |
| # This job generates files dervied from the GPML, including info.json, datanodes.tsv and refs.tsv. | |
| # The meta-data-action Java application performs the bulk of the work: | |
| # https://github.com/wikipathways/meta-data-action | |
| # | |
| # The generated files are uploaded as artifacts to be downloaded and pushed together with other files | |
| # to another repo in a final sync job. This strategy allows for parallel job processing! | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| needs: [get-pr,get-gpml] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| # TODO: impl more efficient way to get scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML artifact | |
| # Downloads gpml-file artifact to working dir | |
| # Note: Automatically unzips as folders containing files with original filenames | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: gpml-file | |
| - name: Setup Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '11' | |
| - name: Cache meta-data-action with dependencies | |
| uses: actions/cache@v3 | |
| id: cacheMetaJar | |
| with: | |
| path: ./meta-data-action-1.1.2-jar-with-dependencies.jar | |
| key: cached-meta-data-action-${{ hashFiles('meta-data-action-1.1.2-jar-with-dependencies.jar') }} | |
| restore-keys: | | |
| cached-meta-data-action-${{ hashFiles('meta-data-action-1.1.2-jar-with-dependencies.jar') }} | |
| cached-meta-data-action- | |
| - name: Install deps | |
| run: | | |
| echo "Refreshing cached-meta-data-action" | |
| if [ ! -e ./meta-data-action-1.1.2-jar-with-dependencies.jar ]; then | |
| wget -O meta-data-action-1.1.2-jar-with-dependencies.jar https://github.com/wikipathways/meta-data-action/releases/download/v1.1.2/meta-data-action-1.1.2-jar-with-dependencies.jar | |
| fi | |
| - name: Cache and install dependencies | |
| uses: actions/cache@v3 | |
| id: cache | |
| with: | |
| path: ${{ github.workspace }}/Hs_Derby_Ensembl_108.bridge | |
| key: ${{ runner.os }}-java-Hs_Derby_Ensembl_108 | |
| restore-keys: | | |
| ${{ runner.os }}-java-Hs_Derby_Ensembl_108 | |
| ${{ runner.os }}-java-Hs_Derby_Ensembl_ | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| name: Install deps | |
| run: | | |
| cd "${{ github.workspace }}" | |
| if [ ! -e ./Hs_Derby_Ensembl_108.bridge ]; then | |
| wget -O Hs_Derby_Ensembl_108.bridge https://zenodo.org/record/7781913/files/Hs_Derby_Ensembl_108.bridge?download=1 | |
| fi | |
| - name: Generate gdb.config, fileNames.config, and fileDownloads.config | |
| run: scripts/meta-data-action/configGenerator.sh ./$GPML_FILE | |
| - name: Cache all bridge files | |
| uses: actions/cache@v3 | |
| id: cacheAllBridge | |
| with: | |
| path: | | |
| ./metabolites*.bridge | |
| ./Ag*.bridge | |
| ./An*.bridge | |
| ./At*.bridge | |
| ./Bs*.bridge | |
| ./Bt*.bridge | |
| ./Ce*.bridge | |
| ./Cf*.bridge | |
| ./Ci*.bridge | |
| ./Dr*.bridge | |
| ./Da*.bridge | |
| ./Dp*.bridge | |
| ./Dm*.bridge | |
| ./Ec*.bridge | |
| ./Gg*.bridge | |
| ./Fg*.bridge | |
| ./Gm*.bridge | |
| ./Hs*.bridge | |
| ./Hv*.bridge | |
| ./Ml*.bridge | |
| ./Mm*.bridge | |
| ./Mx*.bridge | |
| ./Oa*.bridge | |
| ./Ova*.bridge | |
| ./Oi*.bridge | |
| ./Oj*.bridge | |
| ./Pi*.bridge | |
| ./Pt*.bridge | |
| ./Qc*.bridge | |
| ./Rn*.bridge | |
| ./Sc*.bridge | |
| ./Sl*.bridge | |
| ./Ss*.bridge | |
| ./Vv*.bridge | |
| ./Xt*.bridge | |
| ./Zm*.bridge | |
| key: cached-bridge-files | |
| restore-keys: | | |
| cached-bridge-files | |
| - if: steps.cacheAllBridge.outputs.cache-hit != 'true' | |
| name: Install all bridge files | |
| run: | | |
| echo "Cache not found: cached-bridge-files" | |
| declare -a OrganismNames=("Metabolites" "Anopheles gambiae" "Aspergillus niger" "Arabidopsis thaliana" "Bacillus subtilis" "Bos taurus" "Caenorhabditis elegans" "Canis familiaris" "Ciona intestinalis" "Danio rerio" "Daphnia magna" "Daphnia pulex" "Drosophila melanogaster" "Escherichia coli" "Gallus gallus" "Fusarium graminearum" "Glycine max" "Homo sapiens" "Hordeum vulgare" "Macaca mulatta" "Mus musculus" "Mycobacterium tuberculosis" "Ornithorhynchus anatinus" "Ovis aries" "Oryza indica" "Oryza japonica" "Populus trichocarpa" "Pan troglodytes" "Equus caballus" "Rattus norvegicus" "Saccharomyces cerevisiae" "Solanum lycopersicum" "Sus scrofa" "Vitis vinifera" "Xenopus tropicalis" "Zea mays") | |
| for org in "${OrganismNames[@]}"; do | |
| echo "generating configuration files for "$org"" | |
| scripts/meta-data-action/configGenerator.sh "$org" | |
| echo "installing bridgedb files for "$org"" | |
| scripts/meta-data-action/installDependencies.sh "$org" | |
| done | |
| - name: Generate configs, install bridgeDb, generate info and tsv files | |
| run: | | |
| chmod 777 meta-data-action-1.1.2-jar-with-dependencies.jar | |
| f=./$GPML_FILE | |
| org="$(sed -n '/<Pathway /s/.*Organism=\(.*\)[^\n]*/\1/p' $f | tr -d '"' | tr -d '>' | tr -d '\r')" | |
| echo "generating configuration files for "$org"" | |
| scripts/meta-data-action/configGenerator.sh "$org" | |
| wpid="$(echo $GPML_FILE | sed 's/.gpml//')" | |
| cat gdb.config | |
| echo "generating info and datanode files for $wpid, organism "$org"" | |
| java -jar meta-data-action-1.1.2-jar-with-dependencies.jar local "$f" $(date --utc +%F) gdb.config "$org" | |
| # generated files are placed in "pathways" dir | |
| ls -R | |
| mkdir ./metadata | |
| mv pathways/"$wpid"-info.json metadata/. | |
| mv pathways/"$wpid"*.tsv metadata/. | |
| - name: Upload metadata files as artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: metadata | |
| path: metadata/ | |
| retention-days: 1 | |
| if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` | |
| - name: Report on meta-data-action | |
| id: report | |
| run: | | |
| # Verify generated files | |
| infojson=$(find . -path "./metadata/WP*info.json" -print -quit) | |
| dntsv=$(find . -path "./metadata/WP*datanodes.tsv" -print -quit) | |
| refstsv=$(find . -path "./metadata/WP*refs.tsv" -print -quit) | |
| # Update PR description | |
| NEW_DESCRIPTION=" | |
| ## Generate Metadata Files | |
| " | |
| if [[ -n $infojson ]]; then | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [x] info.json generated" | |
| else | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [] info.json generated" | |
| fi | |
| if [[ -n $dntsv ]]; then | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [x] datanodes.tsv generated" | |
| else | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [] datanodes.tsv generated" | |
| fi | |
| if [[ -n $refstsv ]]; then | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [x] refs.tsv generated" | |
| else | |
| NEW_DESCRIPTION="$NEW_DESCRIPTION | |
| - [] refs.tsv generated" | |
| fi | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| authors: | |
| # This job generates new author profiles if needed. It checks the list of authors from the GPML file | |
| # against the list of prior authors in scripts/authors-list.csv and updates the list accordingly. | |
| # | |
| # An artifact is created and uploaded only if new author profiles are generated. A later job will | |
| # check to see if an "authors" artifact is present and commit files at that time. | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML artifact | |
| # Downloads gpml-file artifact to working dir | |
| # Note: Automatically unzips file with original filename | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: gpml-file | |
| - name: Update author list and create md profiles | |
| run: | | |
| # Read list of previous authors | |
| authorList=() | |
| { | |
| read #skip header line | |
| while IFS=, read -r username realname orcid wikidata github; do | |
| authorList+=("$username") | |
| done | |
| }< scripts/author_list.csv | |
| echo ${#authorList[@]} | |
| # Generate unique list of authors from this GPML | |
| uniqueAuthors=() | |
| auth="$(sed -n '/<Pathway /s/.*Author=\"\[\(.*\)\]\".*/\1/p' $GPML_FILE )" | |
| IFS=',' read -r -a curAuthors <<< "$auth" | |
| echo $curAuthors | |
| echo ${#curAuthors[@]} | |
| for i in "${curAuthors[@]}"; do | |
| skip= | |
| for j in "${uniqueAuthors[@]}"; do | |
| [[ $i == $j ]] && { skip=1; break; } | |
| done | |
| [[ -n $skip ]] || uniqueAuthors+=("$i") | |
| done | |
| echo ${uniqueAuthors[@]} | |
| echo ${#uniqueAuthors[@]} | |
| # For any new authors, generate a profile and add them to the list | |
| mkdir ./authors | |
| k=0 | |
| for a in "${uniqueAuthors[@]}"; do | |
| a=$(echo "$a" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') | |
| a2=$(echo "$a" | tr ' ' '_') | |
| if [[ ! " ${authorList[*]} " =~ " ${a} " ]]; then | |
| echo "Adding $a" | |
| echo $a","$a",,," >> scripts/author_list.csv | |
| echo "---" > "authors/$a.md" | |
| echo "username: $a" >> "authors/$a.md" | |
| echo "realname: $a" >> "authors/$a.md" | |
| echo "website: " >> "authors/$a.md" | |
| echo "affiliation: " >> "authors/$a.md" | |
| echo "bio: " >> "authors/$a.md" | |
| echo "github: " >> "authors/$a.md" | |
| echo "orcid: " >> "authors/$a.md" | |
| echo "linkedin: " >> "authors/$a.md" | |
| echo "googlescholar: " >> "authors/$a.md" | |
| echo "wikidata: " >> "authors/$a.md" | |
| echo "twitter: " >> "authors/$a.md" | |
| echo "mastodon-url: " >> "authors/$a.md" | |
| echo "meta:" >> "authors/$a.md" | |
| echo "instagram:" >> "authors/$a.md" | |
| echo "email:" >> "authors/$a.md" | |
| echo "redirect_from:" >> "authors/$a.md" | |
| echo "- /index.php/User:$a2" >> "authors/$a.md" | |
| echo "- /index.php/Special:Contributions/$a2" >> "authors/$a.md" | |
| echo "---" >> "authors/$a.md" | |
| $k=$k + 1 | |
| fi | |
| done | |
| if [ $k -gt 0 ]; then | |
| mv scripts/author_list.csv authors/. | |
| fi | |
| echo "NEW_AUTHORS=$k" >> $GITHUB_ENV | |
| - name: Upload author files as artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: authors | |
| path: authors/ | |
| retention-days: 1 | |
| if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn` | |
| - name: Report on authors | |
| id: report | |
| run: | | |
| # Update PR description | |
| if [ ${{ env.NEW_AUTHORS }} -eq 0 ]; then | |
| NEW_DESCRIPTION="- [x] No new author profiles needed" | |
| elif [ ${{ env.NEW_AUTHORS }} -eq 1 ]; then | |
| NEW_DESCRIPTION="- [x] 1 author profile generated" | |
| else | |
| NEW_DESCRIPTION="- [x] ${{ env.NEW_AUTHORS }} author profiles generated" | |
| fi | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| pubmed: | |
| # This job generates a bibliography.tsv file based on the refs.tsv generated by the metadata job. | |
| # | |
| # A "pubmed" artifact is created and uploaded containing this new tsv file. A later job will | |
| # commit the file to appropriate draft folders in the website repo. | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download Metadata artifact | |
| # Downloads metadata artifact to working dir | |
| # Note: Automatically unzips as a folder containing files with original filenames | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: metadata | |
| - name: Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Cache dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-node- | |
| - name: Install dependencies | |
| working-directory: scripts/generate-references | |
| run: npm install | |
| - name: Generate references | |
| working-directory: scripts/generate-references | |
| run: | | |
| mkdir -p ../../pubmed | |
| cp ../../*-refs.tsv ../../pubmed/. | |
| node index.js | |
| # where are generated files placed | |
| rm ../../*-refs.tsv | |
| - name: Upload pubmed file as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pubmed | |
| path: pubmed/ | |
| retention-days: 1 | |
| if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` | |
| - name: Report on pubmed | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION="- [x] bibliography.tsv generated" | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| frontmatter: | |
| # This job generates a wpid.md file based on the wpid-info.json file generated by the metadata job. | |
| # | |
| # A "frontmatter" artifact is created and uploaded containing this .md file. A later job will | |
| # commit the file to the appropriate draft folder in the website repo. | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download Metadata artifact | |
| # Downloads metadata artifacts to working dir | |
| # Note: Automatically unzips as folders containing files with original filenames | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: metadata | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install deps | |
| run: | | |
| pip install python-frontmatter | |
| - name: Run create_pathway_frontmatter.py | |
| run: | | |
| #f=./$GPML_FILE | |
| wpid="$(echo $GPML_FILE | sed 's/.gpml//')" | |
| echo "generating frontmatter file for $wpid" | |
| json_info_f=./pathways/"$wpid"/"$wpid"-info.json | |
| if [ -e "$json_info_f" ]; then | |
| python scripts/create_pathway_frontmatter.py "$json_info_f" | |
| else | |
| echo "info.json file missing for $wpid" >2 | |
| fi | |
| # generated files are placed in "pathways" dir | |
| ls -R | |
| - name: Report on frontmatter | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION="- [x] .md generated" | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| json-svg: | |
| # This job generates json, svg and png files based on the wpid-info.json file generated by the metadata job. | |
| # | |
| # A "frontmatter" artifact is created and uploaded containing this .md file. A later job will | |
| # commit the file to the appropriate draft folder in the website repo. | |
| # | |
| # A markdown text report is stored as pr-desc to be compiled and written to the PR description. | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML and Metadata artifacts | |
| # Downloads gpml-file and metadata artifacts to working dir | |
| # Note: Automatically unzips as folders containing files with original filenames | |
| uses: actions/download-artifact@v3 | |
| - name: Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Cache NPM dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-node- | |
| - name: Install deps | |
| working-directory: sandbox-wp-db/scripts/generate-svgs | |
| run: | | |
| npm install | |
| sudo apt-get update | |
| sudo apt-get install -y xmlstarlet | |
| - name: Run generate-svgs | |
| run: | | |
| echo "TODO: Run generate-svgs and attach output JSON, SVG and PNG to PR" | |
| - name: Report on pubmed | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION="- [x] .json generated | |
| - [x] .svg generated | |
| - [x] .png generated | |
| *TODO: insert PNG here* | |
| " | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| testing: | |
| if: true | |
| needs: [get-pr,get-gpml,metadata] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr-desc: ${{ steps.report.outputs.pr-desc }} | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| steps: | |
| - name: Checkout repository for scripts | |
| uses: actions/checkout@v4 | |
| - name: Download GPML and Metadata artifacts | |
| # Downloads gpml-file and metadata artifacts to working dir | |
| # Note: Automatically unzips as folders containing files with original filenames | |
| uses: actions/download-artifact@v3 | |
| - name: Perform automatic test | |
| run: | | |
| # TODO: perform testing | |
| - name: Report on pubmed | |
| id: report | |
| run: | | |
| # Update PR description | |
| NEW_DESCRIPTION=' | |
| --- | |
| ## Automated Testing | |
| *As table* | |
| | # | Test name | Link | Notes | Result | | |
| |---|---|---|---|---| | |
| | 1 | Test one | | | \$\${\color{green}PASS}\$\$ | | |
| | 2 | Test two | | | \$\${\color{red}FAIL}\$\$ | | |
| | 3 | Test three | | | \$\${\color{green}PASS}\$\$ | | |
| *As checklist* | |
| - [ ] Interactions are connected | |
| - [ ] Datanodes are annotated with database references | |
| - [ ] Decription, consisting of 2-3 sentence overview of processes described in the pathway | |
| - [ ] At least one literature reference | |
| - [ ] At least one pathway ontology term | |
| - [ ] Pathway title conforms to the [guidelines](https://github.com/wikipathways/wikipathways-faq/discussions/24) | |
| --- | |
| ' | |
| # Store PR description for compilation in final job | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "pr-desc<<$EOF" >> $GITHUB_OUTPUT | |
| echo "$NEW_DESCRIPTION" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| commit-outputs: | |
| needs: [get-pr,get-gpml,metadata,authors,pubmed,frontmatter,json-svg,testing] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all output files from artifacts | |
| uses: actions/download-artifact@v3 | |
| - name: Checkout jekyll repo | |
| # TESTING: change repo and ssh-key from sandbox to real | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: wikipathways/sandbox-wp.gh.io | |
| path: wikipathways.github.io | |
| fetch-depth: 1 | |
| ref: main | |
| ssh-key: ${{ secrets.ACTIONS_SANDBOX_DEPLOY_KEY }} | |
| - name: Commit report | |
| env: | |
| GPML_FILE: ${{ needs.get-gpml.outputs.gpml-file }} | |
| PR_NUMBER: ${{ needs.get-pr.outputs.pr-number }} | |
| run: | | |
| wpid="$(echo $GPML_FILE | sed 's/.gpml//')" | |
| ## Copy to draft_assets | |
| mkdir -p wikipathways.github.io/draft_assets/"$wpid" | |
| cp gpml-file/"$wpid".gpml wikipathways.github.io/draft_assets/"$wpid"/. | |
| cp metadata/"$wpid"*.* wikipathways.github.io/draft_assets/"$wpid"/. | |
| cp pubmed/"$wpid"-bibliography.tsv wikipathways.github.io/draft_assets/"$wpid"/. | |
| #cp json-svg/"$wpid"-bibliography.tsv wikipathways.github.io/draft_assets/"$wpid"/. | |
| ## Copy to _data/drafts | |
| cp metadata/"$wpid"-datanodes.tsv wikipathways.github.io/_data/drafts/. | |
| cp pubmed/"$wpid"-bibliography.tsv wikipathways.github.io/_data/drafts/. | |
| ## Copy to _drafts | |
| cp frontmatter/"$wpid".md wikipathways.github.io/_drafts/. | |
| ## Copy to _authors | |
| if [ -d "authors" ]; then | |
| cp authors/*.md wikipathways.github.io/_authors/. | |
| cp author_list.csv wikipathways.github.io/scripts/. | |
| fi | |
| ## Commit, pull and push | |
| cd "${{ github.workspace }}/wikipathways.github.io" | |
| git config --global user.name 'GitHub Action' | |
| git config --global user.email 'action@github.com' | |
| git add . | |
| if git diff --exit-code --staged; then | |
| echo "No changes" | |
| else | |
| git commit -m 'Adds outputs from PR processing' | |
| git pull --rebase | |
| git push | |
| fi | |
| update-pr-desc: | |
| needs: [get-pr,get-gpml,metadata,authors,pubmed,frontmatter,json-svg,testing] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: refs/pull/${{ needs.get-pr.outputs.pr-number }}/head | |
| - name: Compile and update PR description | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ needs.get-pr.outputs.pr-number }} | |
| run: | | |
| NEW_DESCRIPTION="${{ needs.get-gpml.outputs.pr-desc }} | |
| ${{ needs.metadata.outputs.pr-desc }} | |
| ${{ needs.authors.outputs.pr-desc }} | |
| ${{ needs.pubmed.outputs.pr-desc }} | |
| ${{ needs.frontmatter.outputs.pr-desc }} | |
| ${{ needs.json-svg.outputs.pr-desc }} | |
| ${{ needs.testing.outputs.pr-desc }} | |
| " | |
| # Edit PR description | |
| gh pr edit $PR_NUMBER --body "$NEW_DESCRIPTION" |