mdbook docs with i18n support with github action to upload source file to Crowdin and download translation

.github/typos.toml

@@ -0,0 +1,16 @@
+[default.extend-identifiers]
+# False positives.
+mis = "mis"
+MIS = "MIS"
+inout = "inout"
+BARs = "BARs"
+
+[type.po]
+# Localized content should not be checked for typos. English
+# in these files should be validated manually.
+extend-glob = ["*.po"]
+check-file = false
+
+[files]
+# Typos in third party packages should be fixed upstream.
+extend-exclude = ["theme/*", "tools/*"]

.github/workflows/build.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# Usage: build.sh <book-lang> <dest-dir>
+#
+# Build the docs as of the date specified specified in the
+# POT-Creation-Date header of po/$book_lang.po. The output can be
+# found in $dest_dir.
+#
+# The src/ directory are left in a dirty state so
+# you can run `mdbook test` and other commands afterwards.
+
+book_lang=${1:?"Usage: $0 <book-lang> <dest-dir>"}
+dest_dir=${2:?"Usage: $0 <book-lang> <dest-dir>"}
+
+if [ "$book_lang" = "en" ]; then
+    echo "::group::Building English docs"
+else
+    pot_creation_date=$(grep --max-count 1 '^"POT-Creation-Date:' "po/$book_lang.po" | sed -E 's/".*: (.*)\\n"/\1/')
+    pot_creation_date=${pot_creation_date:-now}
+    echo "::group::Building $book_lang translation as of $pot_creation_date"
+
+    # Set language and adjust site URL. Clear the redirects since they are
+    # in sync with the source files, not the translation.
+    export MDBOOK_BOOK__LANGUAGE=$book_lang
+    export MDBOOK_OUTPUT__HTML__SITE_URL=/ankidroiddocs/$book_lang/
+    export MDBOOK_OUTPUT__HTML__REDIRECT='{}'
+fi
+
+mdbook build -d "$dest_dir"
+
+echo "::endgroup::"

.github/workflows/build.yml

@@ -0,0 +1,102 @@
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install formatting dependencies
+        run: |
+          sudo apt update
+          sudo apt install gettext yapf3
+
+      - name: Check formatting
+        uses: dprint/check@v2.2
+
+  typos:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check for typos
+        uses: crate-ci/typos@v1.17.2
+        with:
+          config: ./.github/typos.toml
+
+  find-languages:
+    runs-on: ubuntu-latest
+    outputs:
+      languages: ${{ steps.find-languages.outputs.languages }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Find languages
+        id: find-languages
+        shell: python
+        run: |
+          import os, json, pathlib
+          languages = ["en"] + [p.stem for p in pathlib.Path("po").iterdir() if p.suffix == ".po"]
+          github_output = open(os.environ["GITHUB_OUTPUT"], "a")
+          github_output.write("languages=")
+          json.dump(sorted(languages), github_output)
+
+  build:
+    runs-on: ubuntu-latest
+    needs:
+      - find-languages
+    strategy:
+      matrix:
+        language: ${{ fromJSON(needs.find-languages.outputs.languages) }}
+      fail-fast: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # We need the full history for build.sh below.
+
+      - name: Setup Rust cache
+        uses: ./.github/workflows/setup-rust-cache
+
+      - name: Install Gettext
+        run: |
+          sudo apt update
+          sudo apt install gettext
+
+      - name: Install mdbook
+        uses: ./.github/workflows/install-mdbook
+
+      - name: Test format of ${{ matrix.language }} translation
+        if: matrix.language != 'en'
+        run: msgfmt --statistics -o /dev/null po/${{ matrix.language }}.po
+
+      - name: Test extracting English strings
+        if: matrix.language == 'en'
+        run: |
+          MDBOOK_OUTPUT='{"xgettext": {"pot-file": "messages.pot"}}' mdbook build -d po
+          msgfmt -o /dev/null --statistics po/messages.pot
+
+      - name: Build ${{ matrix.language }} translation
+        run: |
+          chmod +x .github/workflows/build.sh
+          .github/workflows/build.sh ${{ matrix.language }} book/ankidroiddocs-${{ matrix.language }}
+
+      # Upload the book now to retain it in case mdbook test fails.
+      - name: Upload book
+        uses: actions/upload-artifact@v4
+        with:
+          name: ankidroiddocs-${{ matrix.language }}
+          path: book/
+

.github/workflows/check-msgid-changes.py

@@ -0,0 +1,89 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Find changed msgid fields without a change in POT-Creation-Date.
+
+When following the instructions in
+https://github.com/google/comprehensive-rust/blob/main/TRANSLATIONS.md,
+one of two things should happen:
+
+- The `msgid` fields change because `msgmerge --update` was used. This
+  will also update the POT-Creation-Date field since a new timestamp
+  is merged in from the messages.pot file.
+
+- Translations are added or updated. This should not change the
+  `msgid` fields: only the `msgstr` fields should change. If the PO
+  editor being used inadvertently changes the wrapping of both `msgid`
+  and `msgstr` fields, then `dprint fmt` can be used to normalize them
+  all.
+
+The code here detects if both of these happen at the same time: if one
+or more `msgid` fields changed without a corresponding change to the
+POT-Creation-Date field. If this happens, the translator should fix it
+by running:
+
+  dprint fmt
+
+Commit and push to the branch again.
+"""
+
+import os
+
+# TODO: move the `git reset` from the action code to here. Infact, we
+# should be able to determine this with read-only operations.
+
+# TODO: use Git plumbing commands instead of porcelain, see
+# https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git.html.
+for filename in os.popen("git diff --name-only").read().split():
+    if not filename.endswith(".po"):
+        continue
+
+    # If POT-Creation-Date has changed, then we assume that the commit
+    # is the result of `msgmerge --update`. It is expected that the
+    # `msgid` fields change when `msgmerge` is run, so there is
+    # nothing to check.
+    if "POT-Creation-Date" in os.popen(
+            f"git diff --unified=0 {filename}").read():
+        print(
+            f"Assuming {filename} was changed automatically, skipping msgid check"
+        )
+        continue
+
+    changed_lines = {
+        i + 1
+        for i, line in enumerate(
+            os.popen(f"git blame {filename}").readlines())
+        if line.startswith("00000000")
+    }
+
+    # Look for a changed line between `msgid` and `msgstr`.
+    saw_msgid = False
+    with open(filename, "r") as f:
+        line = f.readline()
+        line_number = 1
+
+        while line:
+            if line.startswith("msgid"):
+                saw_msgid = True
+            elif line.startswith("msgstr"):
+                saw_msgid = False
+
+            if saw_msgid and line_number in changed_lines:
+                print(f"Changed msgid in file {filename}:{line_number}!")
+                print(
+                    "Please read https://github.com/google/comprehensive-rust/blob/main/TRANSLATIONS.md#creating-and-updating-translations."
+                )
+                exit(1)
+
+            line_number += 1
+            line = f.readline()

.github/workflows/check-msgid-changes.yml

@@ -0,0 +1,21 @@
+name: Prevent unintended msgid changes
+
+on:
+  pull_request:
+    paths:
+      - "po/*.po"
+
+jobs:
+  check-msgid-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Reset git
+        run: git reset origin/main
+
+      - name: Check po file changes
+        run: python3 .github/workflows/check-msgid-changes.py

.github/workflows/install-mdbook/action.yml

@@ -0,0 +1,23 @@
+name: Install mdbook and dependencies
+
+description: Install mdbook with the dependencies we need.
+
+runs:
+  using: composite
+  steps:
+    # The --locked flag is important for reproducible builds.
+    - name: Install mdbook
+      run: cargo install mdbook --locked --version 0.4.36
+      shell: bash
+
+    - name: Install mdbook-i18n-helpers
+      run: cargo install mdbook-i18n-helpers --locked --version 0.3.1
+      shell: bash
+
+    - name: Install mdbook-toc
+      run: cargo install mdbook-toc --locked --version 0.14.2
+      shell: bash
+
+    - name: Install mdbook-linkcheck
+      run: cargo install mdbook-linkcheck --locked --version 0.7.7
+      shell: bash

.github/workflows/publish.yml

@@ -1,52 +1,65 @@
 name: Publish
 
 on:
-  workflow_dispatch:
   push:
     branches:
-      - "main"
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow one concurrent deployment
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+  # Update the language picker in index.hbs to link new languages.
+  LANGUAGES: ar ja ru zh-CN
 
 jobs:
-  main:
-    name: "Publish docs.ankidroid.org"
-    if: github.repository == 'ankidroid/ankidroiddocs'
+  publish:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
     steps:
-      - name: SSH key setup
-        uses: webfactory/ssh-agent@v0.8.0
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
-          ssh-private-key: ${{ secrets.DOCS_SSH_PRIVATE_KEY }}
+          fetch-depth: 0 # We need the full history for build.sh below.
 
-      - name: SSH server key setup
-        run: |
-          ssh-keyscan ankidroid.org >> ~/.ssh/known_hosts
-        shell: bash
+      - name: Setup Rust cache
+        uses: ./.github/workflows/setup-rust-cache
 
-      - name: Git Checkout
-        uses: actions/checkout@v3
+      - name: Install mdbook
+        uses: ./.github/workflows/install-mdbook
 
-      - name: Prepare website directory
-        run: |
-          mkdir _site
-          cp -r img _site/
-          cp icons/* _site/
+      - name: Make executable
+        run: chmod +x .github/workflows/build.sh
+
+      - name: Build course in English
+        run: .github/workflows/build.sh en book
 
-      - name: Install Asciidoctor
+      - name: Build all translations
         run: |
-          sudo chown -R $USER /var/lib/gems/
-          sudo chown -R $USER /usr/local/bin
-          gem install asciidoctor
-        shell: bash
-
-      - name: Render Docs
-        run: asciidoctor ./*.asc -D _site/
-
-      - name: Copy Docs to ankidroid.org
-        run: scp -r _site/* ankidroid@ankidroid.org:/usr/share/nginx/html/
-        shell: bash
-
-      - name: Restart nginx web server
-        #  - why? nginx used to fail periodically requiring a restart. But who has access? How to restart?
-        # This alloww people with publish access a button-push way to force a restart if necessary,
-        # and the restart is quick enough that doing it on publish is fine.
-        run: ssh ankidroid@ankidroid.org /usr/bin/sudo /usr/sbin/service nginx restart
+          for po_lang in ${{ env.LANGUAGES }}; do
+              .github/workflows/build.sh $po_lang book/$po_lang
+              mv book/$po_lang/html book/html/$po_lang
+          done
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: book/html
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4