Generate journal-list.mv on build (#10298)

* Generate journal-list.mv on build * Squashed 'buildres/abbrv.jabref.org/' content from commit d47628aa8a git-subtree-dir: buildres/abbrv.jabref.org git-subtree-split: d47628aa8ae2d7d47d5b18a82982d2246996347c * Fix gradle dependencies * Convert buildres/abbrv.jabref.org to git submodule * Convert csl-styles from subtree to submodule * Enable testing at koppor's fork * Fix path of csl-styles and csl-locales * Try to fix branch in sub module * Fix branch name * Add "show-progress: 'false'" (and add some forgotten sub-module checkouts) * Fix path * Fix checkout of submodules * Speed up build * Fix task dependency * Fix dependencies (and exit code) * Fix tabs (by replacing to spaces) * Do not fail if input does not exist (JabRef can cope with that)
JabRef · Sep 3, 2023 · cb88d59 · cb88d59
1 parent 07f54ed
commit cb88d59
Show file tree

Hide file tree

Showing 10,571 changed files with 212 additions and 1,014,295 deletions.
diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml
@@ -8,7 +8,7 @@ permissions:
 jobs:
   dependabot:
     runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
+    if: ${{ (github.actor == 'dependabot[bot]') || (github.actor == 'koppor' && startsWith(github.event.pull_request.title, '[Bot]')) }}
     steps:
       - name: Approve PR
         run: gh pr review --approve "$PR_URL"

diff --git a/.github/workflows/check-links.yaml b/.github/workflows/check-links.yaml
@@ -11,6 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          show-progress: 'false'
       - uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:
           use-quiet-mode: 'yes'

diff --git a/.github/workflows/deployment-arm64.yml b/.github/workflows/deployment-arm64.yml
@@ -49,6 +49,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: 'true'
+          show-progress: 'false'
       - name: Install GitVersion
         uses: gittools/actions/gitversion/setup@v0.10.2
         with:

diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml
@@ -74,6 +74,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: 'true'
+          show-progress: 'false'
       - name: Install pigz and cache (linux)
         if: (matrix.os == 'ubuntu-latest')
         uses: awalsh128/cache-apt-pkgs-action@latest
@@ -273,6 +275,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: 'true'
+          show-progress: 'false'
       - name: Install GitVersion
         if: steps.checksecrets.outputs.secretspresent == 'YES'
         uses: gittools/actions/gitversion/setup@v0.10.2
@@ -334,9 +338,10 @@ jobs:
       - name: Checkout source
         if: steps.checksecrets.outputs.secretspresent == 'YES'
         uses: actions/checkout@v3
-      - name: Fetch all history for all tags and branches
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+          show-progress: 'false'
       - name: Install GitVersion
         if: steps.checksecrets.outputs.secretspresent == 'YES'
         uses: gittools/actions/gitversion/setup@v0.10.2

diff --git a/.github/workflows/gource.yml b/.github/workflows/gource.yml
@@ -12,7 +12,7 @@ on:
 concurrency:
   group: gource
   cancel-in-progress: true
-  
+
 jobs:
   action:
     runs-on: ubuntu-latest
@@ -21,6 +21,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          show-progress: 'false'
       - name: 'Development history of current build'
         uses: BoundfoxStudios/action-gource@v2
         with:

diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml
@@ -28,6 +28,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          show-progress: 'false'
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:

diff --git a/.github/workflows/refresh-csl-subtrees.yml b/.github/workflows/refresh-csl-subtrees.yml
@@ -2,7 +2,6 @@ name: Refresh Citation Style Language Files
 
 on:
   schedule:
-    # run on 1st and 15th of each month
     - cron: '1 2 1,15 * *'
   workflow_dispatch:
 
@@ -21,34 +20,36 @@ jobs:
       - uses: actions/checkout@v3
         with:
           persist-credentials: true
-          ref: main
           fetch-depth: 0
+          submodules: 'true'
+          show-progress: 'false'
       - name: Initialize git
         run: |
-          git checkout main
           git config --local core.editor /usr/bin/cat
           git config user.name 'github-actions[bot]'
           git config user.email 'github-actions[bot]@users.noreply.github.com'
-      - name: Add csl-styles remote
-        run: git remote add -f csl-styles https://github.com/citation-style-language/styles.git
-      - name: Update csl-styles
+      - name: Update buildres/csl/csl-styles
         run: |
-          git subtree pull --prefix buildres/csl/csl-styles csl-styles master --squash || true
+          cd buildres/csl/csl-styles
+          git checkout master
+          git pull
+          cd ..
+          cd ..
+          cd ..
           cp buildres/csl/csl-styles/acm-siggraph.csl src/main/resources/csl-styles/
           cp buildres/csl/csl-styles/apa.csl src/main/resources/csl-styles/
           cp buildres/csl/csl-styles/ieee.csl src/main/resources/csl-styles/
           cp buildres/csl/csl-styles/turabian-author-date.csl src/main/resources/csl-styles/
-          git add .
-          git commit -m"Refresh example styles" || true
-      - name: Add csl-locales remote
-        run: git remote add -f csl-locales https://github.com/citation-style-language/locales.git
-      - name: Update csl-locales
+      - name: Update buildres/csl/csl-locales
         run: |
-          git subtree pull --prefix buildres/csl/csl-locales csl-locales master --squash || true
+          cd buildres/csl/csl-locales
+          git checkout master
+          git pull
+          cd ..
+          cd ..
+          cd ..
           cp buildres/csl/csl-locales/locales.json src/main/resources/csl-locales/
           cp buildres/csl/csl-locales/locales-en-US.xml src/main/resources/csl-locales/
-          git add .
-          git commit -m"Refresh example styles" || true
       - uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/refresh-journal-lists.yml b/.github/workflows/refresh-journal-lists.yml
@@ -1,65 +1,50 @@
 name: Refresh Journal Lists
 
 on:
-  push:
-    paths:
-      - '.github/workflows/refresh-journal-lists.yml'
-      - 'buildSrc/build.gradle'
-      - 'buildSrc/src/main/java/**'
-      - 'src/main/java/org/jabref/logic/journals/**'
+  schedule:
+    - cron: '1 2 2,16 * *'
   workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
   publish:
+    name: Refresh Journal List Files
+    runs-on: ubuntu-latest
     permissions:
       contents: write  # for peter-evans/create-pull-request to create branch
       pull-requests: write  # for peter-evans/create-pull-request to create a PR
-    name: Refresh Journal List Files
-    runs-on: ubuntu-latest
+    if: (github.repository == 'JabRef/jabref' || github.repository == 'koppor/jabref')
     steps:
       - uses: actions/checkout@v3
         with:
           persist-credentials: true
+          fetch-depth: 0
+          submodules: 'true'
+          show-progress: 'false'
+      - name: Initialize git
+        run: |
+          git config --local core.editor /usr/bin/cat
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+      - name: Update buildres/abbrv.jabref.org
+        run: |
+          cd buildres/abbrv.jabref.org
+          git checkout main
+          git pull
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
           java-version: 20
           distribution: 'temurin'
           cache: 'gradle'
-      - name: Update journal lists
+      - name: Check whether journal-list.mv can be generated (the "real" generation is done inside JabRef's build process)
         run: |
-          set +o pipefail
-
-          # prepare journal lists
-          cd /tmp
-          git clone --depth=1 https://github.com/JabRef/abbrv.jabref.org.git
-          cd abbrv.jabref.org/journals
-
-          # remove all lists without dot in them
-          # we use abbreviation lists containing dots in them only (to be consistent)
-          rm journal_abbreviations_entrez.csv
-          rm journal_abbreviations_medicus.csv
-          rm journal_abbreviations_webofscience-dotless.csv
-
-          # we currently do not have good support for BibTeX strings
-          rm journal_abbreviations_ieee_strings.csv
-
-          # prepare building
-          mkdir -p $GITHUB_WORKSPACE/build/journals
-          cp * $GITHUB_WORKSPACE/build/journals/
-
-          # ensure that the .java classes are the most recent ones
-          cp $GITHUB_WORKSPACE/src/main/java/org/jabref/logic/journals/* $GITHUB_WORKSPACE/buildSrc/src/main/java/org/jabref/logic/journals
-
-          # create .mv file
-          cd $GITHUB_WORKSPACE
           ./gradlew generateJournalAbbreviationList
       - uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          branch: update-journallist
+          branch: update-journal-lists
           title: "[Bot] Update journal abbreviation lists"
           commit-message: Update journal abbreviation lists
diff --git a/.github/workflows/tests-fetchers.yml b/.github/workflows/tests-fetchers.yml
@@ -42,6 +42,9 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          show-progress: 'false'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -29,6 +29,9 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v3
+        with:
+          submodules: 'false'
+          show-progress: 'false'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -77,6 +80,9 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          show-progress: 'false'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -109,6 +115,9 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          show-progress: 'false'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -143,6 +152,9 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          show-progress: 'false'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -181,6 +193,9 @@ jobs:
           SNAPCRAFT_LOGIN_FILE: ${{ secrets.CODECOV_TOKEN }}
       - name: Checkout source
         uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          show-progress: 'false'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -201,12 +216,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          show-progress: 'false'
       - uses: gradle/wrapper-validation-action@v1
   # This ensures that no git merge conflict markers (<<<, ...) are contained
   merge_conflict_job:
     name: Find merge conflicts
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          show-progress: 'false'
       - name: Merge Conflict finder
         uses: olivernybroe/action-conflict-finder@v4.0
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,9 @@
+[submodule "buildres/abbrv.jabref.org"]
+  path = buildres/abbrv.jabref.org
+  url = https://github.com/JabRef/abbrv.jabref.org.git
+[submodule "buildres/csl-styles"]
+  path = buildres/csl/csl-styles
+  url = https://github.com/citation-style-language/styles.git
+[submodule "buildres/csl-locales"]
+  path = buildres/csl/csl-locales
+  url = https://github.com/citation-style-language/locales.git
diff --git a/build.gradle b/build.gradle
@@ -1,5 +1,4 @@
 import org.gradle.internal.os.OperatingSystem
-import org.jabref.build.JournalAbbreviationConverter
 import org.jabref.build.xjc.XjcPlugin
 import org.jabref.build.xjc.XjcTask
 
@@ -302,6 +301,14 @@ tasks.register("generateSearchGrammarSource", JavaExec) {
     args  = ["-o","src-gen/main/java/org/jabref/search" , "-visitor", "-no-listener", "-package", "org.jabref.search", "$projectDir/src/main/antlr4/org/jabref/search/Search.g4"]
 }
 
+tasks.register("generateJournalAbbreviationList", JavaExec) {
+    group = "JabRef"
+    description = "Converts the comma-separated journal abbreviation file to a H2 MVStore"
+    classpath = sourceSets.main.runtimeClasspath
+    main = "org.jabref.cli.JournalListMvGenerator"
+}
+jar.dependsOn "generateJournalAbbreviationList"
+test.dependsOn "generateJournalAbbreviationList"
 
 tasks.register('generateCitaviSource', XjcTask) {
     group = 'JabRef'
@@ -312,17 +319,6 @@ tasks.register('generateCitaviSource', XjcTask) {
     javaPackage = "org.jabref.logic.importer.fileformat.citavi"
 }
 
-tasks.register('generateJournalAbbreviationList', JournalAbbreviationConverter) {
-    group = 'JabRef'
-    description = "Converts the comma-separated journal abbreviation file to a H2 MVStore."
-
-    // This directory is filled by "refresh-journal-lists.yml"
-    // It takes the contents of https://github.com/JabRef/abbrv.jabref.org.git
-    inputDir = file("build/journals")
-
-    outputDir = file("src/main/resources/journals")
-}
-
 tasks.withType(JavaCompile).configureEach {
     options.encoding = 'UTF-8'
 }

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle