diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/android/android_strings_file_writer.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/android/android_strings_file_writer.rb new file mode 100644 index 000000000..ad9ecaea3 --- /dev/null +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/android/android_strings_file_writer.rb @@ -0,0 +1,29 @@ +require 'fastlane_core/ui/ui' +require 'fileutils' + +module Fastlane + module Helper + module Android + module StringsFileWriter + # @param [String] dir path to destination directory + # @param [Locale] locale the locale to write the file for + # @param [File, IO] io The File IO containing the translations downloaded from GlotPress + def self.write_app_translations_file(dir:, locale:, io:) + # `dir` is typically `src/main/res/` here + return unless Locale.valid?(locale, :android) + + dest = File.join(dir, locale.android_path) + FileUtils.mkdir_p(File.dirname(dest)) + + # TODO: reorder XML nodes alphabetically, for easier diffs + # xml = Nokogiri::XML(io, nil, Encoding::UTF_8.to_s) + # # … reorder nodes … + # File.open(main, 'w:UTF-8') { |f| f.write(xml.to_xml(indent: 4)) } + # FIXME: For now, just copy blindly until we get time to implement node reordering + UI.message("Writing: #{dest}") + IO.copy_stream(io, dest) + end + end + end + end +end diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/fastlane_metadata_file_writer.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/fastlane_metadata_file_writer.rb new file mode 100644 index 000000000..b928ef784 --- /dev/null +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/fastlane_metadata_file_writer.rb @@ -0,0 +1,102 @@ +require 'fastlane_core/ui/ui' +require 'fileutils' + +module Fastlane + module Helper + module FastlaneMetadataFilesWriter + + # A model/struct defining a rule on how to process and map metadata from GlotPress into txt files + # + # @param [String] key The key in the GlotPress export for the metadata + # @param [Int] max_len The maximum length allowed by the App Store / Play Store for that key. + # Note: If the translation for `key` exceeds the specified `max_len`, we will try to find an alternate key named `#{key}_short` by convention. + # @param [String] filename The (relative) path to the `.txt` file to write that metadata to + # + MetadataRule = Struct.new(:key, :max_len, :filename) do + # The common standardized set of Metadata rules for an Android project + def self.android_rules(version_name:, version_code:) + suffix = version_name.gsub('.', '') + [ + MetadataRule.new("release_note_#{suffix}", 500, File.join('changelogs', "#{version_code}.txt")), + MetadataRule.new('play_store_app_title', 30, 'title.txt'), + MetadataRule.new('play_store_promo', 80, 'short_description.txt'), + MetadataRule.new('play_store_desc', 4000, 'full_description.txt'), + ] + end + + # The common standardized set of Metadata rules for an Android project + def self.ios_rules(version_name:) + suffix = version_name.gsub('.', '') + [ + MetadataRule.new("release_note_#{suffix}", 4000, 'release_notes.txt'), + MetadataRule.new('app_store_name', 30, 'name.txt'), + MetadataRule.new('app_store_subtitle', 30, 'subtitle.txt'), + MetadataRule.new('app_store_description', 4000, 'description.txt'), + MetadataRule.new('app_store_keywords', 100, 'keywords.txt'), + ] + end + end + + # Visit each key/value pair of a translations Hash, and yield keys and matching translations from it based on the passed `MetadataRules`, + # trying any potential fallback key if the translation exceeds the max limit, and yielding each found and valid entry to the caller. + # + # @param [#read] io + # @param [Array] rules List of rules for each key + # @param [Block] rule_for_unknown_key An optional block called when a key that does not match any of the rules is encountered. + # The block will receive a [String] (key) and must return a `MetadataRule` instance (or nil) + # + # @yield [String, MetadataRule, String] yield each (key, matching_rule, value) tuple found in the JSON, after resolving alternates for values exceeding max length + # Note that if both translations for the key and its (optional) shorter alternate exceeds the max_len, it will still `yield` but with a `nil` value + # + def self.visit(translations:, rules:, rule_for_unknown_key:) + translations.each do |key, value| + next if key.nil? || key.end_with?('_short') # skip if alternate key + + rule = rules.find { |r| r.key == key } + rule = rule_for_unknown_key.call(key) if rule.nil? && !rule_for_unknown_key.nil? + next if rule.nil? + + if rule.max_len != nil && value.length > rule.max_len + UI.warning "Translation for #{key} is too long (#{value.length}), trying shorter alternate #{key}." + short_key = "#{key}_short" + value = json[short_key] + if value.nil? + UI.warning "No shorter alternate (#{short_key}) available, skipping entirely." + yield key, rule, nil + next + end + if value.length > rule.max_len + UI.warning "Translation alternate for #{short_key} was too long too (#{value.length}), skipping entirely." + yield short_key, rule, nil + next + end + end + yield key, rule, value + end + end + + # Write the `.txt` files to disk for the given exported translation file (typically a JSON export) based on the `MetadataRules` provided + # + # @param [String] locale_dir the path to the locale directory (e.g. `fastlane/metadata/android/fr`) to write the `.txt` files to + # @param [Hash] translations The hash of translations (key => translation) to visit based on `MetadataRules` then write to disk. + # @param [Array] rules The list of fixed `MetadataRule` to use to extract the expected metadata from the `translations` + # @param [Block] rule_for_unknown_key An optional block called when a key that does not match any of the rules is encountered. + # The block will receive a [String] (key) and must return a `MetadataRule` instance (or nil) + # + def self.write(locale_dir:, translations:, rules:, &rule_for_unknown_key) + self.visit(translations: translations, rules: rules, rule_for_unknown_key: rule_for_unknown_key) do |_key, rule, value| + dest = File.join(locale_dir, rule.filename) + if value.nil? && File.exist?(dest) + # Key found in JSON was rejected for being too long. Delete file + UI.verbose("Deleting file #{dest}") + FileUtils.rm(dest) + elsif value + UI.verbose("Writing file #{dest}") + FileUtils.mkdir_p(File.dirname(dest)) + File.write(dest, value.chomp) + end + end + end + end + end +end diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/glotpress_downloader.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/glotpress_downloader.rb new file mode 100755 index 000000000..cd50b5f37 --- /dev/null +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/glotpress_downloader.rb @@ -0,0 +1,93 @@ +require 'fastlane_core/ui/ui' +require 'json' +require 'open-uri' +require 'zip' + +module Fastlane + module Helper + class GPDownloader + REQUEST_HEADERS = { 'User-Agent' => Wpmreleasetoolkit::USER_AGENT } + + module FORMAT + ANDROID = 'android' + IOS = 'strings' + JSON = 'json' + end + + # The host of the GlotPress instance. e.g. `'translate.wordpress.org'` + attr_accessor :host + # The path of the project in GlotPress. e.g. `'apps/ios/release-notes'` + attr_accessor :project + + def initialize(host:, project:) + @host = host + @project = project + end + + # @param [String] gp_locale + # @param [String] format Typically `'android'`, `'strings'` or `'json'` + # @param [Hash] filters + # + # @yield [IO] the corresponding downloaded IO content + # + # @note For this case, `project_url` is on the form 'https://translate.wordpress.org/projects/apps/ios/release-notes' + def download_locale(gp_locale:, format:, filters: { status: 'current'}) + query_params = filters.transform_keys { |k| "filters[#{k}]" }.merge(format: format) + uri = URI::HTTPS.build(host: host, path: File.join('/', 'projects', project, gp_locale, 'default', 'export-translations'), query: URI.encode_www_form(query_params)) + + UI.message "Downloading #{uri}" + io = begin + uri.open(REQUEST_HEADERS) + rescue StandardError => e + UI.error "Error downloading #{gp_locale} - #{e.message}" + return + end + UI.message "Download done." + yield io + end + + # @param [String] format Typically `'android'`, `'strings'` or `'json'` + # @param [Hash] filters + # + # @yield For each locale, a tuple of [String], [IO] corresponding to the glotpress locale code and IO content + # + # @note requires the GlotPress instance to have the Bulk Downloader plugin installed + # @note For this case, `project_url` is on the form 'https://translate.wordpress.org/exporter/apps/android/dev/' + def download_all_locales(format:, filters: { status: 'current'}) + query_params = filters.transform_keys { |k| "filters[#{k}]" }.merge('export-format': format) + uri = URI::HTTPS.build(host: host, path: File.join('/', 'exporter', project, '-do'), query: URI.encode_www_form(query_params)) + UI.message "Downloading #{uri}" + zip_stream = uri.open(REQUEST_HEADERS) + UI.message "Download done." + + Zip::File.open_buffer(zip_stream) do |zip_file| + zip_file.each do |entry| + next if entry.name.end_with?('/') && entry.size.zero? + + prefix = File.dirname(entry.name).gsub(/[0-9-]*$/, '') + '-' + locale = File.basename(entry.name, File.extname(entry.name)).delete_prefix(prefix) + UI.message "- Found locale in ZIP: #{locale}" + + yield locale, entry.get_input_stream + end + end + end + + # Takes a GlotPress JSON export and transform it to a simple `Hash` of key => value pairs + # + # Since the JSON format for GlotPress exports is a bit odd, with JSON keys actually being a concatenation of actual + # copy key and source copy, and values being an array, this allows us to convert this odd export format to a more + # usable structure. + # + # @param [#read] io The `File` or `IO` to read the JSON data exported from GlotPress + def parse_json_export(io:) + json = JSON.parse(io.read) + json.map do |composite_key, values| + key = composite_key.split(/\u0004/).first # composite_key is a concatenation of key + \u0004 + source] + value = values.first # Each value in the JSON Hash is an Array of all the translations; but if we provided the right filter, the first one should always be the right one + [key, value] + end.to_h + end + end # class + end # module +end # module diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/ios/ios_strings_file_writer.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/ios/ios_strings_file_writer.rb new file mode 100644 index 000000000..9d9a37d06 --- /dev/null +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/ios/ios_strings_file_writer.rb @@ -0,0 +1,24 @@ +require 'fastlane_core/ui/ui' +require 'fileutils' + +module Fastlane + module Helper + module Ios + module StringsFileWriter + # @param [String] dir path to destination directory + # @param [Locale] locale the locale to write the file for + # @param [File, IO] io The File IO containing the translations downloaded from GlotPress + def self.write_app_translations_file(dir:, locale:, io:) + # `dir` is typically `WordPress/Resources/` here + return unless Locale.valid?(locale, :ios) + + dest = File.join(dir, locale.ios_path) + FileUtils.mkdir_p(File.dirname(dest)) + UI.message("Writing: #{dest}") + IO.copy_stream(io, dest) + end + end + end + end +end + diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/models/locale.rb b/lib/fastlane/plugin/wpmreleasetoolkit/models/locale.rb new file mode 100644 index 000000000..3b57c0b3a --- /dev/null +++ b/lib/fastlane/plugin/wpmreleasetoolkit/models/locale.rb @@ -0,0 +1,29 @@ +require 'fastlane_core/ui/ui' + +module Fastlane + module Wpmreleasetoolkit + Locale = Struct.new(:glotpress, :android, :playstore, :ios, :appstore, keyword_init: true) do + def android_path + File.join("values-#{self.android}", 'strings.xml') + end + + def ios_path + File.join("#{self.ios}.lproj", 'Localizable.strings') + end + + def self.valid?(locale, *keys) + if locale.nil? + UI.warning("Locale is unknown") + return false + end + keys.each do |key| + if locale[key].nil? + UI.warning("Locale #{locale} is missing required key #{key}") + return false + end + end + return true + end + end + end +end diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/models/locales.rb b/lib/fastlane/plugin/wpmreleasetoolkit/models/locales.rb new file mode 100644 index 000000000..80c365bae --- /dev/null +++ b/lib/fastlane/plugin/wpmreleasetoolkit/models/locales.rb @@ -0,0 +1,152 @@ +module Fastlane + module Wpmreleasetoolkit + class Locales + ALL_KNOWN_LOCALES = [ + Locale.new(glotpress: 'ar', android: 'ar', playstore: 'ar'), + Locale.new(glotpress: 'de', android: 'de', playstore: 'de-DE'), + Locale.new(glotpress: 'en-gb', android: 'en-rGB', playstore: 'en-US'), + Locale.new(glotpress: 'es', android: 'es', playstore: 'es-ES'), + Locale.new(glotpress: 'fr-ca', android: 'fr-rCA', playstore: 'fr-CA'), + Locale.new(glotpress: 'fr', android: 'fr', playstore: 'fr-FR', ios: 'fr-FR', appstore: 'fr-FR'), + Locale.new(glotpress: 'he', android: 'he', playstore: 'iw-IL'), + Locale.new(glotpress: 'id', android: 'id', playstore: 'id'), + Locale.new(glotpress: 'it', android: 'it', playstore: 'it-IT'), + Locale.new(glotpress: 'ja', android: 'ja', playstore: 'ja-JP'), + Locale.new(glotpress: 'ko', android: 'ko', playstore: 'ko-KR'), + Locale.new(glotpress: 'nl', android: 'nl', playstore: 'nl-NL'), + Locale.new(glotpress: 'pl', android: 'pl', playstore: 'pl-PL'), + Locale.new(glotpress: 'pt-br', android: 'pt-rBR', playstore: 'pt-BR', ios: 'pt-BR', appstore: 'pt-BR'), + Locale.new(glotpress: 'ru', android: 'ru', playstore: 'ru-RU'), + Locale.new(glotpress: 'sr', android: 'sr', playstore: 'sr'), + Locale.new(glotpress: 'sv', android: 'sv', playstore: 'sv-SE'), + Locale.new(glotpress: 'th', android: 'th', playstore: 'th'), + Locale.new(glotpress: 'tr', android: 'tr', playstore: 'tr-TR'), + Locale.new(glotpress: 'vi', android: 'vi', playstore: 'vi'), + Locale.new(glotpress: 'zh-cn', android: 'zh-rCN', playstore: 'zh-CN', ios: 'zh-Hans', appstore: 'zh-Hans'), + Locale.new(glotpress: 'zh-tw', android: 'zh-rTW', playstore: 'zh-TW', ios: 'zh-Hant', appstore: 'zh-Hant'), + Locale.new(glotpress: 'az', android: 'az'), + Locale.new(glotpress: 'el', android: 'el') + # FIXME: Complete the list with ios/app_store properties for all, and extending to more locales + ] + + MAG16_GP_CODES = %w[ar de es fr he id it ja ko nl pt-br ru sv tr zh-cn zh-tw].freeze + + ############## + + # [Array] + attr_accessor :locales + + # @param [Array,Array] locales + def initialize(locales = ALL_KNOWN_LOCALES) + @locales = locales.map { |l| l.is_a?(Locale) ? l : Locale.new(l) } + end + + ############## + # @!group Filter `Locales` based on locale codes + + # Return the list of locales matching the gp_codes passed as input parameters + # + # @param [String...] codes The locale codes to get the Locales for + # @param [Symbol] key_name The name of the `Locale` property to use to filter those locales by. + # Defaults to `:glotpress` (= the `codes` param is expected to be _GlotPress_ locale codes by default) + # @return [Locales] + def self.[](*codes, key_name: :glotpress) + locales = ALL_KNOWN_LOCALES.select { |l| codes.include?(l[key_name.to_sym]) } + Locales.new(locales) + end + + # Find a single given locale amongst the set of all known locales + # + # @param [String] code + # @param [Symbol] key_name The name of the `Locale` property to use to filter those locales by. + # Defaults to `:glotpress` (= the `codes` param is expected to be _GlotPress_ locale codes by default) + # @return [Locale?] The known locale matching the provided code, or `nil` if no known locale was found. + def self.find(code, key_name: :glotpress) + Locales.all.find(code, key_name: key_name) + end + + # Find a single given locale amongst the set of locales registered in this `Locales` instance + # + # @param [String] code + # @param [Symbol] key_name The name of the `Locale` property to use to filter those locales by. + # Defaults to `:glotpress` (= the `codes` param is expected to be _GlotPress_ locale codes by default) + # @return [Locale?] The known locale matching the provided code, or `nil` if no known locale was found. + def find(code, key_name: :glotpress) + @locales.find { |l| code == l[key_name.to_sym] } + end + + # @!endgroup + ############## + + ############## + # @!group Common locale sets + + def self.all + Locales.new(ALL_KNOWN_LOCALES) + end + + def self.mag16 + Locales[*MAG16_GP_CODES] + end + + # @!endgroup + ############## + + ############## + # @!group Locales set arithmetics + + # Substraction + def -(other) + Locales.new(self.locales - other.locales) + end + + # Intersection + def &(other) + Locales.new(self.locales & other.locales) + end + + # Addition (without deduplication guarantee) + def +(other) + Locales.new(self.locales + other.locales) + end + + # Union (with deduplication) + def |(other) + Locales.new(self.locales | other.locales) + end + + # @!endgroup + ############## + + ############## + # @!group Conversion to other types and iteration + + def each + @locales.each { |l| yield l } + end + + # Constructs a `Hash` whose keys are the locale code for `key_sym` (e.g. `:glotpress`) and corresponding values are the locale code for `value_sym` (e.g. `:android`) + # Example: `Locales.mag16.to_hash(:glotpress, :android)` + def to_hash(key_sym, value_sym) + Hash.new( + @locales.map { |l| [l[key_sym], l[value_sym]] } + ) + end + + def to_a + if block_given? + @locales.map { |l| yield l } + else + @locales + end + end + + def to_s + "\#" + end + + # @!endgroup + ############## + end + end +end diff --git a/spec/glotpress_downloader_spec.rb b/spec/glotpress_downloader_spec.rb new file mode 100644 index 000000000..4e325fc58 --- /dev/null +++ b/spec/glotpress_downloader_spec.rb @@ -0,0 +1,98 @@ +# NOTE: This is not really a spec but a demo script instead that I used to test my implementation. +# FIXME: Convert this to an actual spec with unit test cases and stubs/fixtures + +module GlotpressDownloaderDemo + DOTORG = 'translate.wordpress.org' + DOTCOM = 'translate.wordpress.com' + WP_ANDROID = { host: DOTORG, project: 'apps/android/dev' } + WP_IOS = { host: DOTORG, project: 'apps/ios/dev' } + WC_ANDROID = { host: DOTCOM, project: 'woocommerce/woocommerce-android' } + WC_IOS = { host: DOTCOM, project: 'woocommerce/woocommerce-ios' } + + EXPORT_FMT = Fastlane::Helper::GPDownloader::FORMAT + FastlaneMetadataFilesWriter = Fastlane::Helper::FastlaneMetadataFilesWriter + + EXAMPLE_OUTPUT_DIR = 'MyTestApp' + + # Example Usages for App Translation + + def demo_android_app_translations_bulk + output_dir = File.join(EXAMPLE_OUTPUT_DIR, 'src', 'main', 'res') + FileUtils.mkdir_p(output_dir) + downloader = Fastlane::Helper::GPDownloader.new(**WC_ANDROID) + downloader.download_all_locales(format: EXPORT_FMT::ANDROID) do |gp_locale, io| + locale = Locales.all.find(gp_locale) + Fastlane::Helper::Android::StringsFileWriter.write(dir: output_dir, locale: locale, io: io) unless locale.nil? # skip unknown locales that may be present in ZIP + end + end + + def demo_ios_app_translations_bulk + output_dir = File.join(EXAMPLE_OUTPUT_DIR, 'Resources') + FileUtils.mkdir_p(output_dir) + downloader = Fastlane::Helper::GPDownloader.new(**WC_IOS) + downloader.download_all_locales(format: EXPORT_FMT::IOS) do |gp_locale, io| + locale = Locales.all.find(gp_locale) + Fastlane::Helper::Ios::StringsFileWriter.write(dir: output_dir, locale: locale, io: io) unless locale.nil? # skip unknown locales that may be present in ZIP + end + end + + def demo_ios_app_translations_loop + output_dir = File.join(EXAMPLE_OUTPUT_DIR, 'Resources') + FileUtils.mkdir_p(output_dir) + downloader = Fastlane::Helper::GPDownloader.new(**WC_IOS) + Locales.mag16.each do |locale| + downloader.download_locale(gp_locale: locale.glotpress, format: EXPORT_FMT::IOS) do |io| + Fastlane::Helper::Ios::StringsFileWriter.write(dir: output_dir, locale: locale, io: io) + end + end + end + + # Example Usages for Metadata + + def demo_android_metadata_bulk + downloader = Fastlane::Helper::GPDownloader.new(host: DOTORG, project: 'apps/android/release-notes') + downloader.download_all_locales(format: EXPORT_FMT::JSON) do |gp_locale, io| + locale = Locales.mag16.find(gp_locale) + next unless Locale.valid?(locale, :playstore) + + rules = FastlaneMetadataFilesWriter::MetadataRule.android_rules(version_name: '20.4', version_code: 1234) + translations = downloader.class.parse_json_export(io: io) # Convert odd GlotPress JSON export format to standard Hash + + locale_dir = File.join(EXAMPLE_OUTPUT_DIR, 'fastlane', 'metadata', 'android', locale.playstore) + FastlaneMetadataFilesWriter.write(locale_dir: locale_dir, translations: translations, rules: rules) do |key| + # Example: if we find a non-standard key which ends up being a screenshot key, save under screenshots/ subdir. + # Otherwise, just ignore any other unknown key. + if key.start_with?('play_store_screenshot_') + FastlaneMetadataFilesWriter::MetadataRule.new(key, nil, File.join('screenshots', "#{key.delete_prefix('play_store_screenshot_')}.txt")) + end + end + end + end + + def demo_android_metadata_loop + downloader = Fastlane::Helper::GPDownloader.new(host: DOTORG, project: 'apps/android/release-notes/') + Locales['fr', 'es'].each do |locale| + next unless Locale.valid?(locale, :playstore) + downloader.download_locale(gp_locale: locale.glotpress, format: EXPORT_FMT::JSON) do |io| + locale_dir = File.join(EXAMPLE_OUTPUT_DIR, 'fastlane', 'metadata', 'android', locale.playstore) + rules = FastlaneMetadataFilesWriter::MetadataRule.android_rules(version_name: '20.4', version_code: 1234) + translations = downloader.class.parse_json_export(io: io) # Convert odd GlotPress JSON export format to standard Hash + puts "Writing to #{locale_dir}..." + FastlaneMetadataFilesWriter.write(locale_dir: locale_dir, translations: translations, rules: rules) + end + end + end + + def demo_ios_metadata_loop + downloader = Fastlane::Helper::GPDownloader.new(host: DOTORG, project: 'apps/ios/release-notes/') + Locales.mag16.each do |locale| + next unless Locale.valid?(locale, :appstore) + downloader.download_locale(gp_locale: locale.glotpress, format: EXPORT_FMT::JSON) do |io| + locale_dir = File.join(EXAMPLE_OUTPUT_DIR, 'fastlane', 'metadata', locale.appstore) + rules = FastlaneMetadataFilesWriter::MetadataRule.ios_rules(version_name: '20.4') + translations = downloader.class.parse_json_export(io: io) # Convert odd GlotPress JSON export format to standard Hash + FastlaneMetadataFilesWriter.write(locale_dir: locale_dir, translations: translations, rules: rules) + end + end + end +end