build: Move translations pull to Docker build time

This allows people to get fresher translations than the previous system which involved pulling them weekly into this repo and only releasing new translations every with Aspects releases.

tutoraspects/templates/aspects/build/aspects-superset/Dockerfile

@@ -8,6 +8,8 @@ FROM apache/superset:4.0.1
 USER root
 
 COPY ./requirements.txt /app/requirements.txt
+COPY ./localization/compile_translations.py /app/localization/compile_translations.py
+COPY ./localization/datasets_strings.yaml /app/localization/datasets_strings.yaml
 
 RUN pip install -r /app/requirements.txt
 
@@ -18,6 +20,7 @@ ARG FIREFOX_VERSION=117.0.1
 
 RUN apt-get update -q \
     && apt-get install -yq --no-install-recommends \
+        git \
         libnss3 \
         libdbus-glib-1-2 \
         libgtk-3-0 \
@@ -33,6 +36,11 @@ RUN apt-get update -q \
     && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* /var/[log,tmp]/* /tmp/* && apt-get clean
 
 COPY ./openedx-assets /app/openedx-assets
-COPY ./localization/ /app/localization/
+
+# Pull latest aspects translations from openedx-tranlsations repository
+RUN atlas pull translations/tutor-contrib-aspects/tutoraspects/templates/aspects/apps/superset/conf/locale:/app/localization/
+
+# combine the /app/localization/ files into a single `localization/locale.yaml` file in a way Aspects can use
+RUN python /app/localization/compile_translations.py
 
 USER superset

tutoraspects/templates/aspects/build/aspects-superset/localization/compile_translations.py

@@ -0,0 +1,71 @@
+import os
+import glob
+import shutil
+
+import ruamel.yaml
+import ruamel.yaml.comments
+
+yaml = ruamel.yaml.YAML()
+
+
+def recursive_sort_mappings(s):
+    """Given a ruamel yaml object, recursively sort all mappings in order."""
+    if isinstance(s, list):
+        for elem in s:
+            recursive_sort_mappings(elem)
+        return
+    if not isinstance(s, dict):
+        return
+    for key in sorted(s, reverse=True):
+        value = s.pop(key)
+        recursive_sort_mappings(value)
+        s.insert(0, key, value)
+
+def compile_translations():
+    """
+    Combine translated files into the single file we use for translation.
+
+    This should be called after we pull translations using Atlas, see the
+    pull_translations make target.
+    """
+    translations_path = "/app/localization/"
+
+    all_translations = ruamel.yaml.comments.CommentedMap()
+    yaml_files = glob.glob(os.path.join(translations_path, "**/locale.yaml"))
+
+    for file_path in yaml_files:
+        lang = file_path.split(os.sep)[-2]
+        with open(file_path, "r", encoding="utf-8") as asset_file:
+            loc_str = asset_file.read()
+        loaded_strings = yaml.load(loc_str)
+
+        # Sometimes translated files come back with "en" as the top level
+        # key, but still translated correctly.
+        try:
+            all_translations[lang] = loaded_strings[lang]
+        except KeyError:
+            all_translations[lang] = loaded_strings["en"]
+
+        if None in all_translations[lang]:
+            all_translations[lang].pop(None)
+
+    out_path = "/app/localization/locale.yaml"
+
+    print(f"Writing all translations out to {out_path}")
+    with open(out_path, "w", encoding="utf-8") as outfile:
+        outfile.write("---\n")
+        # If we don't use an extremely large width, the jinja in our translations
+        # can be broken by newlines. So we use the largest number there is.
+        recursive_sort_mappings(all_translations)
+        yaml.dump(all_translations, outfile)
+        outfile.write("\n{{ patch('superset-extra-asset-translations')}}\n")
+
+    # We remove these files to avoid confusion about where translations are coming
+    # from, and because otherwise we will need to re-save them with the large
+    # width as above to avoid Jinja parsing errors.
+    print("Removing downloaded translations files... ")
+    # shutil.rmtree(translations_path)
+
+
+if __name__ == "__main__":
+    compile_translations()

tutoraspects/templates/aspects/build/aspects-superset/requirements.txt

@@ -1,5 +1,6 @@
 authlib  # OAuth2
 clickhouse-connect>0.5,<1.0
-sentry-sdk[flask]
+openedx-atlas
 ruamel-yaml==0.18.6
+sentry-sdk[flask]
 urllib3>=1.26.15,<2

tutoraspects/templates/base-docker-compose-services

@@ -6,7 +6,6 @@ image: {{ DOCKER_IMAGE_SUPERSET }}
     - ../../env/plugins/aspects/apps/superset/security:/app/security
     - ../../env/plugins/aspects/apps/superset/superset_home:/app/superset_home
     - ../../env/plugins/aspects/apps/superset/scripts:/app/scripts
-    - ../../env/plugins/aspects/build/aspects-superset/localization:/app/localization
     - ../../env/plugins/aspects/build/aspects-superset/openedx-assets:/app/openedx-assets
   restart: unless-stopped
   environment:

tutoraspects/translations/translate_utils.py

@@ -163,57 +163,6 @@ def mark_text_for_translation(asset):
     return []
 
 
-def compile_translations(root_path):
-    """
-    Combine translated files into the single file we use for translation.
-
-    This should be called after we pull translations using Atlas, see the
-    pull_translations make target.
-    """
-    translations_path = os.path.join(
-        root_path, "tutoraspects/templates/aspects/apps/superset/conf/locale"
-    )
-
-    all_translations = ruamel.yaml.comments.CommentedMap()
-    yaml_files = glob.glob(os.path.join(translations_path, "**/locale.yaml"))
-
-    for file_path in yaml_files:
-        lang = file_path.split(os.sep)[-2]
-        with open(file_path, "r", encoding="utf-8") as asset_file:
-            loc_str = asset_file.read()
-        loaded_strings = yaml.load(loc_str)
-
-        # Sometimes translated files come back with "en" as the top level
-        # key, but still translated correctly.
-        try:
-            all_translations[lang] = loaded_strings[lang]
-        except KeyError:
-            all_translations[lang] = loaded_strings["en"]
-
-        if None in all_translations[lang]:
-            all_translations[lang].pop(None)
-
-    out_path = os.path.join(
-        root_path,
-        "tutoraspects/templates/aspects/build/aspects-superset/localization/locale.yaml",
-    )
-
-    print(f"Writing all translations out to {out_path}")
-    with open(out_path, "w", encoding="utf-8") as outfile:
-        outfile.write("---\n")
-        # If we don't use an extremely large width, the jinja in our translations
-        # can be broken by newlines. So we use the largest number there is.
-        recursive_sort_mappings(all_translations)
-        yaml.dump(all_translations, outfile)
-        outfile.write("\n{{ patch('superset-extra-asset-translations')}}\n")
-
-    # We remove these files to avoid confusion about where translations are coming
-    # from, and because otherwise we will need to re-save them with the large
-    # width as above to avoid Jinja parsing errors.
-    print("Removing downloaded translations files... ")
-    shutil.rmtree(translations_path)
-
-
 def extract_translations(root_path):
     """
     This gathers all translatable text from the Superset assets.