Merge pull request #197 from Webperf-se/issue-196

make it easier to generate .mo files

.github/workflows/verify_result.py

@@ -133,6 +133,9 @@ def validate_po_file(locales_dir, localeName, languageSubDirectory, file, msg_id
             # for every .mo file found, try to load it to verify it works
             n_of_errors = 0
             try:
+                # NOTE: gettext is internally caching all mo files, we need to clear this variable to readd the newly generated .mo file.
+                gettext._translations = {}
+
                 language = gettext.translation(
                     file.replace('.po', ''), localedir=locales_dir, languages=[localeName])
                 language.install()
@@ -165,20 +168,20 @@ def validate_po_file(locales_dir, localeName, languageSubDirectory, file, msg_id
                     if lang_txt != msg_txt:
                         print(
                             '  ## Text missmatch:')
-                        print('- msgid: {0}'.format(msg_id))
+                        print('  - msgid: {0}'.format(msg_id))
                         if len(msg_txt) > 15:
                             print(
-                                '  - expected text: "{0}[...]"'.format(msg_txt[0: 15]))
+                                '    - expected text: "{0}[...]"'.format(msg_txt[0: 15]))
                         else:
                             print(
-                                '  - expected text: "{0}"'.format(msg_txt))
+                                '    - expected text: "{0}"'.format(msg_txt))
 
                         if len(lang_txt) > 15:
                             print(
-                                '  - recived text:  "{0}[...]"'.format(lang_txt[0:15]))
+                                '    - recived text:  "{0}[...]"'.format(lang_txt[0:15]))
                         else:
                             print(
-                                '  - recived text:  "{0}"'.format(lang_txt))
+                                '    - recived text:  "{0}"'.format(lang_txt))
                         n_of_errors += 1
                         continue
                 if n_of_errors > 0:
@@ -310,6 +313,27 @@ def validate_locales(dir, msg_ids):
                 # for every .po file found, check if we have a .mo file
                 if validate_po_file(locales_dir, localeName, languageSubDirectory, file, msg_ids):
                     current_number_of_valid_translations += 1
+                elif file.endswith('.po'):
+                    # po file had errors, try generate new mo file and try again.
+                    msgfmt_path = find_msgfmt_py()
+                    if msgfmt_path != None:
+                        print(
+                            '  - Trying to generate .mo file so it matches .po file')
+                        bashCommand = "python {0} -o {1} {2}".format(
+                            msgfmt_path, os.path.join(languageSubDirectory, file.replace(
+                                '.po', '.mo')), os.path.join(languageSubDirectory, file))
+                        import subprocess
+                        process = subprocess.Popen(
+                            bashCommand.split(), stdout=subprocess.PIPE)
+                        output, error = process.communicate()
+                        result = str(output)
+                        if validate_po_file(locales_dir, localeName, languageSubDirectory, file, msg_ids):
+                            current_number_of_valid_translations += 1
+                        else:
+                            is_valid = False
+                    else:
+                        print(
+                            '  - Unable to generate .mo file because we could not find msgfmt.py in python installation')
                 else:
                     is_valid = False
 
@@ -330,6 +354,42 @@ def validate_locales(dir, msg_ids):
     return is_valid
 
 
+def find_msgfmt_py():
+    import sys
+    for python_path in sys.path:
+        a = python_path
+
+        if a.endswith('.zip'):
+            # Ignore zip files
+            continue
+
+        msgfmt_path = has_dir_msgfmt_py(a, 0)
+        if msgfmt_path != None:
+            return msgfmt_path
+
+    return None
+
+
+def has_dir_msgfmt_py(dir, depth):
+    try:
+        files = os.listdir(dir)
+
+        if 'msgfmt.py' in files:
+            return os.path.join(dir, 'msgfmt.py')
+        elif 'i18n' in files:
+            return os.path.join(dir, 'i18n' 'msgfmt.py')
+        elif 'Tools' in files:
+            return os.path.join(dir, 'Tools', 'i18n', 'msgfmt.py')
+        elif 'io.py' in files or 'base64.py' in files or 'calendar.py' in files or 'site-packages' in files:
+            parent_dir = Path(os.path.dirname(
+                os.path.realpath(dir)) + os.path.sep).parent
+            return has_dir_msgfmt_py(parent_dir, depth + 1)
+
+    except Exception as ex:
+        print('\t   Exception', ex)
+    return None
+
+
 def main(argv):
     """
     WebPerf Core - Regression Test

docs/CONTRIBUTING.md

@@ -38,14 +38,15 @@ You could either follow the more technical suggestions below, or you perhaps wou
 
 To create a new language source file:  
 ```python3 <your path to pygettext.py > -d webperf-core -o locales/webperf-core.pot default.py checks.py```
+(or copy an existing one)
 
 Copy the file to your locale, for Swedish it would be:  
 ```locales/sv/LC_MESSAGES/webperf-core.pot```
 
 Rename the file extension from `.pot` to `.po`
 
-After you have translated everything you should run the following command from the LC_MESSAGES folder:  
-```python3 <your path to msgfmt.py> -o webperf-core.mo webperf-core.po```
+After you have translated everything you should run the following command from webperf-core root folder:  
+```python .github\workflows\verify_result.py -l```
 
 You now have support for a new language, please send it to the official repository using a pull request :)
 
@@ -59,6 +60,9 @@ It might be as follows:
 
 #### How to find msgfmt.py
 
+This step is not needed anymore, just run `verify_result.py` command described above.
+IF it doesn't work you can `i18n` folder to python environment variables.
+
 Locate your msgfmt.py file:  
 ```locate msgfmt.py```