po-refresh

#!/usr/bin/env python3

# This file is part of Cockpit.
#
# Copyright (C) 2017 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

import os
import json
import glob
import subprocess
import sys

sys.dont_write_bytecode = True

import task
from machine.machine_core.directories import BASE_DIR


def run(context, verbose=False, **kwargs):
    cwd = BASE_DIR

    def output(*args):
        if verbose:
            sys.stderr.write("+ " + " ".join(args) + "\n")
        return subprocess.check_output(args, cwd=cwd, universal_newlines=True)

    def output_with_stderr(*args):
        if verbose:
            sys.stderr.write("+ " + " ".join(args) + "\n")
        return subprocess.check_output(args, cwd=cwd, universal_newlines=True, stderr=subprocess.STDOUT)

    def execute(*args):
        if verbose:
            sys.stderr.write("+ " + " ".join(args) + "\n")
        subprocess.check_call(args, cwd=cwd)

    def changed(filename):
        lines = output("git", "diff", "--", filename).split("\n")
        msgstr = False
        for line in lines:
            if line.startswith("+msgstr ") and not line.endswith('""'):
                if verbose:
                    sys.stderr.write("{0}: {1}\n".format(filename, line[8:]))
                msgstr = True
        return msgstr

    # Make a tree with source and run "update-po" in it... outputs working directory
    line = output("bots/make-source", "update-po")
    work = os.path.abspath(line.strip())

    # Do the various updates
    cmd = ["make", "upload-pot", "clean-po", "download-po"]
    if verbose:
        sys.stderr.write("+ " + " ".join(cmd) + "\n")
    subprocess.check_call(cmd, cwd=work)

    local_branch = None
    user = None
    current_manifest = None
    current_linguas = []
    language_map = {}
    # LINGUAS file in use and therefore needs updating
    linguas_exists = os.path.isfile("po/LINGUAS")
    # Manifest file is use - and for its updating we also need language map
    manifest_exists = os.path.isfile("pkg/shell/manifest.json.in") and os.path.isfile("po/language_map.txt")

    # Get locale from language-map file. Fail with keyError when not found
    def get_locale(file_name):
        language = os.path.splitext(os.path.basename(file_name))[0]
        if manifest_exists:
            return language_map[language]
        return [language]

    def add_and_commit_lang(name, language, action):
        git_cmd = ["git", "add", "--"]
        if linguas_exists:
            with open("po/LINGUAS", "w", encoding='utf-8') as lngs:
                print("\n".join(current_linguas), file=lngs)
            git_cmd.append("po/LINGUAS")
        if manifest_exists:
            with open("pkg/shell/manifest.json.in", "w", encoding='utf-8') as mnfst:
                text = json.dumps(current_manifest, ensure_ascii=False, indent=4)
                print(text, file=mnfst)
            git_cmd.append("pkg/shell/manifest.json.in")
        execute(*git_cmd, name)
        return task.branch(context, "po: {0} '{1}' language".format(action, language),
                           pathspec=None, branch=local_branch, push=False, **kwargs)

    # Build language map a read manifest
    if manifest_exists:
        with open("po/language_map.txt", "r") as lm:
            for line in lm:
                line = line.strip()
                if not line:
                    continue
                items = line.split(":")
                language_map[items[0]] = items

        # Read manifest
        with open("pkg/shell/manifest.json.in", "r", encoding='utf-8') as mnfst:
            current_manifest = json.load(mnfst)

    # Read linguas
    if linguas_exists:
        with open("po/LINGUAS", "r", encoding='utf-8') as lngs:
            current_linguas = lngs.read().strip().split()

    # Remove all files that have less than 50% coverage
    for po in glob.glob("po/*.po"):
        all_types = output_with_stderr("msgfmt", "--statistics", po).split(", ")
        translated = int(all_types[0].split(" ")[0])
        untranslated = 0
        for u in all_types[1:]:
            untranslated += int(u.split(" ")[0])
        coverage = translated / (translated + untranslated)
        if coverage < 0.5:
            output("rm", po)

    # Remove languages that fall under 50% translated
    files = output("git", "ls-files", "--deleted", "po/")
    for name in files.splitlines():
        if name.endswith(".po"):
            locale = get_locale(name)
            if current_manifest:
                current_manifest["locales"].pop(locale[2])
            if current_linguas:
                current_linguas.remove(locale[0])
            (user, local_branch) = add_and_commit_lang(name, locale[0], "Drop").split(":")

    # Add languages that got over 50% translated
    files = output("git", "ls-files", "--others", "--exclude-standard", "po/")
    for name in files.splitlines():
        if name.endswith(".po"):
            locale = get_locale(name)
            if current_manifest:
                current_manifest["locales"][locale[2]] = locale[1]
                current_manifest["locales"] = dict(sorted(current_manifest["locales"].items()))
            if current_linguas:
                current_linguas.append(locale[0])
                current_linguas.sort()
            (user, local_branch) = add_and_commit_lang(name, locale[0], "Add").split(":")

    # Here we have logic to only include files that actually
    # changed translations, and reset all the remaining ones
    files = output("git", "ls-files", "--modified", "po/")
    for name in files.splitlines():
        if name.endswith(".po"):
            if changed(name):
                execute("git", "add", "--", name)
            else:
                execute("git", "checkout", "--", name)

    # Create a pull request from these changes
    branch = task.branch(context, "po: Update from Fedora Zanata", pathspec="po/",
                         branch=local_branch, **kwargs)

    if branch:
        task.pull(branch, **kwargs)
    elif local_branch:
        if kwargs["issue"]:
            clean = "https://github.com/{0}".format(task.find_our_fork(user))
            task.comment_done(kwargs["issue"], "po-refresh", clean, local_branch, context)

        task.push_branch(user, local_branch)
        task.pull("{0}:{1}".format(user, local_branch), **kwargs)


if __name__ == '__main__':
    task.main(function=run, title="Update translations from Fedora Zanata")