From 6f067ab26ad34ce200377c423003a873a2400b3d Mon Sep 17 00:00:00 2001 From: James Gilles Date: Sun, 5 May 2019 19:10:30 -0400 Subject: [PATCH 1/3] Add i18n_runtime to askama_shared. --- .travis.yml | 1 + askama_shared/Cargo.toml | 7 + askama_shared/src/error.rs | 20 ++ askama_shared/src/i18n_runtime.rs | 322 ++++++++++++++++++++++++++++++ askama_shared/src/lib.rs | 13 +- 5 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 askama_shared/src/i18n_runtime.rs diff --git a/.travis.yml b/.travis.yml index 343cc636d..19775e972 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ before_script: script: - cargo test --all +- cd askama_shared && cargo test --features full && cd .. - if [[ "${TRAVIS_RUST_VERSION}" == stable ]]; then cd testing && cargo test --features full && cargo fmt -- --check; fi diff --git a/askama_shared/Cargo.toml b/askama_shared/Cargo.toml index 79a5a31b0..f9a13bd8c 100644 --- a/askama_shared/Cargo.toml +++ b/askama_shared/Cargo.toml @@ -9,6 +9,10 @@ license = "MIT/Apache-2.0" workspace = ".." edition = "2018" +[features] +full = ["with-i18n", "serde_json", "serde_yaml"] +with-i18n = ["fluent-bundle", "lazy_static", "accept-language"] + [dependencies] askama_escape = { version = "0.2.0", path = "../askama_escape" } humansize = "1.1.0" @@ -18,3 +22,6 @@ serde_derive = "1.0" serde_json = { version = "1.0", optional = true } serde_yaml = { version = "0.8", optional = true } toml = "0.5" +fluent-bundle = { version = "0.6", optional = true } +lazy_static = { version = "1.3", optional = true } +accept-language = { version = "2.0", optional = true } diff --git a/askama_shared/src/error.rs b/askama_shared/src/error.rs index f3e8b4830..133a2bebf 100644 --- a/askama_shared/src/error.rs +++ b/askama_shared/src/error.rs @@ -36,6 +36,14 @@ pub enum Error { #[cfg(feature = "serde_yaml")] Yaml(::serde_yaml::Error), + /// internationalization error from fluent + #[cfg(feature = "with-i18n")] + I18n(::fluent_bundle::errors::FluentError), + + /// missing locale error + #[cfg(feature = "with-i18n")] + NoTranslationsForMessage(String), + /// This error needs to be non-exhaustive as /// the `Json` variants existence depends on /// a feature. @@ -49,6 +57,12 @@ impl ErrorTrait for Error { Error::Fmt(ref err) => err.description(), #[cfg(feature = "serde_json")] Error::Json(ref err) => err.description(), + #[cfg(feature = "with-fluent")] + // fluent uses failure, don't want to bring all that in ;-; + Error::I18n(_) => "fluent i18n error", + #[cfg(feature = "with-fluent")] + // fluent uses failure, don't want to bring all that in ;-; + Error::NoTranslationsForMessage(_) => "missing translations for message", _ => "unknown error: __Nonexhaustive", } } @@ -73,6 +87,12 @@ impl Display for Error { Error::Json(ref err) => write!(formatter, "json conversion error: {}", err), #[cfg(feature = "serde_yaml")] Error::Yaml(ref err) => write!(formatter, "yaml conversion error: {}", err), + #[cfg(feature = "with-fluent")] + Error::I18n(ref err) => write!(formatter, "fluent i18n error: {}", err), + #[cfg(feature = "with-fluent")] + Error::NoTranslationsForMessage(message) => { + write!("missing translations error for message `{:?}`", message) + } _ => write!(formatter, "unknown error: __Nonexhaustive"), } } diff --git a/askama_shared/src/i18n_runtime.rs b/askama_shared/src/i18n_runtime.rs new file mode 100644 index 000000000..0b0c8b73a --- /dev/null +++ b/askama_shared/src/i18n_runtime.rs @@ -0,0 +1,322 @@ +//! Code used in the implementation of `impl_localize!` and the `localize()` filter. +//! +//! Everything in this module should be considered an internal implementation detail; it is only public +//! for use by the macro. +//! +//! Maintenance note: in general, the policy is to move as much i18n code as possible into here; +//! whatever absolutely *must* be included in the generated code is done in askama_derive. + +use accept_language::parse as accept_language_parse; +use fluent_bundle::{FluentBundle, FluentResource, FluentValue}; +use std::collections::HashMap; + +use super::{Error, Result}; + +pub use lazy_static::lazy_static; + +/// An I18n argument value. Instantiated only by the `{ localize() }` filter. +pub type I18nValue = fluent_bundle::FluentValue; + +/// A known locale. Instantiated only by the `impl_localize!` macro. +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)] +pub struct Locale(pub &'static str); + +/// Sources; an array mapping Locales to fluent source strings. Instantiated only by the `impl_localize!` macro. +pub type Sources = &'static [(Locale, &'static str)]; + +/// Sources that have been parsed. Instantiated only by the `impl_localize!` macro. +/// +/// This type is initialized in a lazy_static! in impl_localize!, +/// because FluentBundle can only take FluentResources by reference; +/// we have to store them somewhere to reference them. +/// This can go away once https://github.com/projectfluent/fluent-rs/issues/103 lands. +pub struct Resources(Vec<(Locale, FluentResource)>); + +impl Resources { + /// Parse a list of sources into a list of resources. + pub fn new(sources: Sources) -> Resources { + Resources( + sources + .iter() + .map(|(locale, source)| { + ( + *locale, + FluentResource::try_new(source.to_string()) + .expect("baked .ftl translation failed to parse"), + ) + }) + .collect(), + ) + } +} + +/// StaticParser is a type that handles accessing the translations baked into +/// the output executable / library easy. Instantiated only by the `impl_localize!` macro. +pub struct StaticParser<'a> { + /// Bundles used for localization. + /// Maps long-form locales (e.g. "en_US", not just "en") to their respective bundles. + bundles: HashMap>, + + /// A listing of available locales. + /// Long-form locales map to themselves ("en_US" => [Locale("en_US")]); + /// Short-form locales map to all available long-form locales, in alphabetical order: + /// ("en" => [Locale("en_UK"), Locale("en_US")]). + locales: HashMap<&'static str, Vec>, + + /// The default locale chosen if no others can be determined. + default_locale: Locale, +} + +impl<'a> StaticParser<'a> { + /// Create a StaticParser. + pub fn new(resources: &'a Resources, default_locale: Locale) -> StaticParser<'a> { + assert!( + resources + .0 + .iter() + .find(|(locale, _)| *locale == default_locale) + .is_some(), + "default locale not available!" + ); + + let mut bundles = HashMap::new(); + let mut locales = HashMap::new(); + for (locale, resource) in resources.0.iter() { + // confusingly, this value is used by fluent for number and date formatting only. + // we have to implement looking up missing messages in other bundles ourselves. + let fallback_chain = &[locale.0]; + + let mut bundle = FluentBundle::new(fallback_chain); + + bundle + .add_resource(resource) + .expect("failed to add resource"); + bundles.insert(*locale, bundle); + locales.insert(locale.0, vec![*locale]); + + let short = &locale.0[..2]; + let shorts = locales.entry(short).or_insert_with(|| vec![]); + shorts.push(*locale); + // ensure determinism in fallback order + shorts.sort(); + } + + StaticParser { + bundles, + locales, + default_locale, + } + } + + /// Creates a chain of locales to use for message lookups. + /// * `user_locales`: a list of locales allowed by the user, + /// in descending order of preference. + /// - May be empty. + /// - May be short-form locales (e.g. "en") + /// * `accept_language`: an `Accept-Language` header, if present. + pub fn create_locale_chain( + &self, + user_locales: &[&str], + accept_language: Option<&str>, + ) -> Vec { + let mut chain = vec![]; + + // when adding a locale "en_AU", also check its short form "en", + // and also add all locales that that short form maps to. + // this ensures that a locale chain like "es-AR", "en-US" will + // pull messages from "es-MX" before going to english. + // + // note: this has the side effect of discarding some ordering information. + // + // TODO: discuss whether this is a reasonable approach. + let mut add = |locale_code: &str| { + let mut codes = &[locale_code, &locale_code[..2]][..]; + if locale_code.len() == 2 { + codes = &codes[..1] + } + + for code in codes { + if let Some(locales) = self.locales.get(code) { + for locale in locales { + if !chain.contains(locale) { + chain.push(*locale); + } + } + } + } + }; + for locale_code in user_locales { + add(locale_code); + } + if let Some(accept_language) = accept_language { + for locale_code in &accept_language_parse(accept_language) { + add(locale_code); + } + } + + if !chain.contains(&self.default_locale) { + chain.push(self.default_locale); + } + + chain + } + + /// Localize a message. + /// * `locale_chain`: a list of locales, in descending order of preference + /// * `message`: a message ID + /// * `args`: a slice of arguments to pass to Fluent. + pub fn localize( + &self, + locale_chain: &[Locale], + message: &str, + args: &[(&str, &FluentValue)], + ) -> Result { + let args = if args.len() == 0 { + None + } else { + Some(args.into_iter().map(|(k, v)| (*k, (*v).clone())).collect()) + }; + let args = args.as_ref(); + + for locale in locale_chain { + let bundle = self.bundles.get(locale); + let bundle = if let Some(bundle) = bundle { + bundle + } else { + // TODO warn? + continue; + }; + // this API is weirdly awful; + // format returns Option<(String, Vec)> + // which we have to cope with + let result = bundle.format(message, args); + + if let Some((result, errs)) = result { + if errs.len() == 0 { + return Ok(result); + } else { + continue; + + // TODO: fluent degrades gracefully; maybe just warn here? + // Err(Error::I18n(errs.pop().unwrap())) + } + } + } + // nowhere to fall back to + Err(Error::NoTranslationsForMessage(format!( + "no translations for message {} in locale chain {:?}", + message, locale_chain + ))) + } + + pub fn has_message(&self, locale_chain: &[Locale], message: &str) -> bool { + locale_chain + .iter() + .flat_map(|locale| self.bundles.get(locale)) + .any(|bundle| bundle.has_message(message)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SOURCES: Sources = &[ + ( + Locale("en_US"), + r#" +greeting = Hello, { $name }! You are { $hours } hours old. +goodbye = Goodbye. +"#, + ), + ( + Locale("en_AU"), + r#" +greeting = G'day, { $name }! You are { $hours } hours old. +goodbye = Hooroo. +"#, + ), + ( + Locale("es_MX"), + r#" +greeting = ¡Hola, { $name }! Tienes { $hours } horas. +goodbye = Adiós. +"#, + ), + ( + Locale("de_DE"), + "greeting = Hallo { $name }! Du bist { $hours } Stunden alt.", + ), + ]; + + #[test] + fn basic() -> Result<()> { + let resources = Resources::new(SOURCES); + let bundles = StaticParser::new(&resources, Locale("en_US")); + let name = FluentValue::from("Jamie"); + let hours = FluentValue::from(190321.31); + let args = &[("name", &name), ("hours", &hours)][..]; + + assert_eq!( + bundles.localize(&[Locale("en_US")], "greeting", args)?, + "Hello, Jamie! You are 190321.31 hours old." + ); + assert_eq!( + bundles.localize(&[Locale("es_MX")], "greeting", args)?, + "¡Hola, Jamie! Tienes 190321.31 horas." + ); + assert_eq!( + bundles.localize(&[Locale("de_DE")], "greeting", args)?, + "Hallo Jamie! Du bist 190321.31 Stunden alt." + ); + + // missing messages should fall back to first available + assert_eq!( + bundles.localize( + &[Locale("de_DE"), Locale("es_MX"), Locale("en_US")], + "goodbye", + &[] + )?, + "Adiós." + ); + + if let Ok(_) = bundles.localize(&[Locale("en_US")], "bananas", &[]) { + panic!("Should return Err on missing message"); + } + + Ok(()) + } + + #[test] + fn create_locale_chain() { + let resources = Resources::new(SOURCES); + let bundles = StaticParser::new(&resources, Locale("en_US")); + + // accept-language parser works + short-code lookup works + assert_eq!( + bundles.create_locale_chain(&[], Some("en_US, es_MX; q=0.5")), + &[Locale("en_US"), Locale("en_AU"), Locale("es_MX")] + ); + + // first choice has precedence + assert_eq!( + bundles.create_locale_chain(&["es_MX"], Some("en_US; q=0.5")), + &[Locale("es_MX"), Locale("en_US"), Locale("en_AU")] + ); + + // short codes work + assert_eq!( + bundles.create_locale_chain(&[], Some("en")), + &[Locale("en_AU"), Locale("en_US")] + ); + + // default works + assert_eq!(bundles.create_locale_chain(&[], None), &[Locale("en_US")]); + + // missing languages fall through to default + assert_eq!( + bundles.create_locale_chain(&["zh_HK"], Some("xy_ZW")), + &[Locale("en_US")] + ); + } +} diff --git a/askama_shared/src/lib.rs b/askama_shared/src/lib.rs index ffc035102..a5e9810ce 100644 --- a/askama_shared/src/lib.rs +++ b/askama_shared/src/lib.rs @@ -2,6 +2,11 @@ #[macro_use] extern crate serde_derive; +#[cfg(feature = "with-i18n")] +extern crate accept_language; +#[cfg(feature = "with-i18n")] +extern crate fluent_bundle; + use toml; use std::collections::HashSet; @@ -19,6 +24,9 @@ use std::collections::BTreeMap; pub mod filters; pub mod helpers; +#[cfg(feature = "with-i18n")] +pub mod i18n_runtime; + #[derive(Debug)] pub struct Config<'a> { pub dirs: Vec, @@ -404,7 +412,10 @@ mod tests { vec![ (str_set(&["js"]), "::askama::Js".into()), (str_set(&["html", "htm", "xml"]), "::askama::Html".into()), - (str_set(&["md", "none", "txt", "yml", ""]), "::askama::Text".into()), + ( + str_set(&["md", "none", "txt", "yml", ""]), + "::askama::Text".into() + ), (str_set(&["j2", "jinja", "jinja2"]), "::askama::Html".into()), ] ); From d39d8483ee1c758036818e1dbc874d477aefa46b Mon Sep 17 00:00:00 2001 From: James Gilles Date: Mon, 6 May 2019 18:59:55 -0400 Subject: [PATCH 2/3] Switch to fluent_locale for negotiation, rearrange runtime --- askama_shared/Cargo.toml | 4 +- askama_shared/src/i18n_runtime.rs | 209 +++++++++++++----------------- askama_shared/src/lib.rs | 4 +- 3 files changed, 97 insertions(+), 120 deletions(-) diff --git a/askama_shared/Cargo.toml b/askama_shared/Cargo.toml index f9a13bd8c..854c2545c 100644 --- a/askama_shared/Cargo.toml +++ b/askama_shared/Cargo.toml @@ -11,7 +11,7 @@ edition = "2018" [features] full = ["with-i18n", "serde_json", "serde_yaml"] -with-i18n = ["fluent-bundle", "lazy_static", "accept-language"] +with-i18n = ["fluent-bundle", "fluent-locale", "lazy_static"] [dependencies] askama_escape = { version = "0.2.0", path = "../askama_escape" } @@ -23,5 +23,5 @@ serde_json = { version = "1.0", optional = true } serde_yaml = { version = "0.8", optional = true } toml = "0.5" fluent-bundle = { version = "0.6", optional = true } +fluent-locale = { version = "0.4", optional = true } lazy_static = { version = "1.3", optional = true } -accept-language = { version = "2.0", optional = true } diff --git a/askama_shared/src/i18n_runtime.rs b/askama_shared/src/i18n_runtime.rs index 0b0c8b73a..a34cbfd96 100644 --- a/askama_shared/src/i18n_runtime.rs +++ b/askama_shared/src/i18n_runtime.rs @@ -6,70 +6,35 @@ //! Maintenance note: in general, the policy is to move as much i18n code as possible into here; //! whatever absolutely *must* be included in the generated code is done in askama_derive. -use accept_language::parse as accept_language_parse; use fluent_bundle::{FluentBundle, FluentResource, FluentValue}; -use std::collections::HashMap; +use fluent_locale::{negotiate_languages, parse_accepted_languages, NegotiationStrategy}; +use std::collections::{HashMap, HashSet}; use super::{Error, Result}; pub use lazy_static::lazy_static; -/// An I18n argument value. Instantiated only by the `{ localize() }` filter. -pub type I18nValue = fluent_bundle::FluentValue; - -/// A known locale. Instantiated only by the `impl_localize!` macro. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)] -pub struct Locale(pub &'static str); - -/// Sources; an array mapping Locales to fluent source strings. Instantiated only by the `impl_localize!` macro. -pub type Sources = &'static [(Locale, &'static str)]; - -/// Sources that have been parsed. Instantiated only by the `impl_localize!` macro. -/// -/// This type is initialized in a lazy_static! in impl_localize!, -/// because FluentBundle can only take FluentResources by reference; -/// we have to store them somewhere to reference them. -/// This can go away once https://github.com/projectfluent/fluent-rs/issues/103 lands. -pub struct Resources(Vec<(Locale, FluentResource)>); - -impl Resources { - /// Parse a list of sources into a list of resources. - pub fn new(sources: Sources) -> Resources { - Resources( - sources - .iter() - .map(|(locale, source)| { - ( - *locale, - FluentResource::try_new(source.to_string()) - .expect("baked .ftl translation failed to parse"), - ) - }) - .collect(), - ) - } -} - /// StaticParser is a type that handles accessing the translations baked into /// the output executable / library easy. Instantiated only by the `impl_localize!` macro. pub struct StaticParser<'a> { /// Bundles used for localization. /// Maps long-form locales (e.g. "en_US", not just "en") to their respective bundles. - bundles: HashMap>, + bundles: HashMap<&'static str, FluentBundle<'a>>, - /// A listing of available locales. - /// Long-form locales map to themselves ("en_US" => [Locale("en_US")]); - /// Short-form locales map to all available long-form locales, in alphabetical order: - /// ("en" => [Locale("en_UK"), Locale("en_US")]). - locales: HashMap<&'static str, Vec>, + /// Available locales. + available: Vec<&'static str>, + + /// Optimization: we always treat locales as &'static strs; this is used + /// to convert &'a strs to &'static strs + available_set: HashSet<&'static str>, /// The default locale chosen if no others can be determined. - default_locale: Locale, + default_locale: &'static str, } impl<'a> StaticParser<'a> { /// Create a StaticParser. - pub fn new(resources: &'a Resources, default_locale: Locale) -> StaticParser<'a> { + pub fn new(resources: &'a Resources, default_locale: &'static str) -> StaticParser<'a> { assert!( resources .0 @@ -80,11 +45,11 @@ impl<'a> StaticParser<'a> { ); let mut bundles = HashMap::new(); - let mut locales = HashMap::new(); + let mut available = Vec::new(); for (locale, resource) in resources.0.iter() { // confusingly, this value is used by fluent for number and date formatting only. // we have to implement looking up missing messages in other bundles ourselves. - let fallback_chain = &[locale.0]; + let fallback_chain = &[locale]; let mut bundle = FluentBundle::new(fallback_chain); @@ -92,18 +57,17 @@ impl<'a> StaticParser<'a> { .add_resource(resource) .expect("failed to add resource"); bundles.insert(*locale, bundle); - locales.insert(locale.0, vec![*locale]); - let short = &locale.0[..2]; - let shorts = locales.entry(short).or_insert_with(|| vec![]); - shorts.push(*locale); - // ensure determinism in fallback order - shorts.sort(); + available.push(*locale); } + available.sort(); + + let available_set = available.iter().cloned().collect(); StaticParser { bundles, - locales, + available, + available_set, default_locale, } } @@ -118,47 +82,33 @@ impl<'a> StaticParser<'a> { &self, user_locales: &[&str], accept_language: Option<&str>, - ) -> Vec { - let mut chain = vec![]; - - // when adding a locale "en_AU", also check its short form "en", - // and also add all locales that that short form maps to. - // this ensures that a locale chain like "es-AR", "en-US" will - // pull messages from "es-MX" before going to english. - // - // note: this has the side effect of discarding some ordering information. - // - // TODO: discuss whether this is a reasonable approach. - let mut add = |locale_code: &str| { - let mut codes = &[locale_code, &locale_code[..2]][..]; - if locale_code.len() == 2 { - codes = &codes[..1] - } - - for code in codes { - if let Some(locales) = self.locales.get(code) { - for locale in locales { - if !chain.contains(locale) { - chain.push(*locale); - } - } - } - } + ) -> Vec<&'static str> { + let requested = accept_language.map(|accept_language| { + let mut accepted = user_locales.to_owned(); + accepted.extend(&parse_accepted_languages(accept_language)); + accepted + }); + let requested = match requested { + Some(ref requested) => &requested[..], + None => user_locales, }; - for locale_code in user_locales { - add(locale_code); - } - if let Some(accept_language) = accept_language { - for locale_code in &accept_language_parse(accept_language) { - add(locale_code); - } - } - - if !chain.contains(&self.default_locale) { - chain.push(self.default_locale); - } + let result = negotiate_languages( + requested, + &self.available, + Some(self.default_locale), + &NegotiationStrategy::Filtering, + ); - chain + // prove to borrowck that all locales are static strings + result + .into_iter() + .map(|l| { + *self + .available_set + .get(l) + .expect("invariant violated: available and available_set have same contents") + }) + .collect() } /// Localize a message. @@ -167,7 +117,7 @@ impl<'a> StaticParser<'a> { /// * `args`: a slice of arguments to pass to Fluent. pub fn localize( &self, - locale_chain: &[Locale], + locale_chain: &[&'static str], message: &str, args: &[(&str, &FluentValue)], ) -> Result { @@ -209,7 +159,7 @@ impl<'a> StaticParser<'a> { ))) } - pub fn has_message(&self, locale_chain: &[Locale], message: &str) -> bool { + pub fn has_message(&self, locale_chain: &[&'static str], message: &str) -> bool { locale_chain .iter() .flat_map(|locale| self.bundles.get(locale)) @@ -217,34 +167,65 @@ impl<'a> StaticParser<'a> { } } +/// Sources that have been parsed. Instantiated only by the `impl_localize!` macro. +/// +/// This type is initialized in a lazy_static! in impl_localize!, +/// because FluentBundle can only take FluentResources by reference; +/// we have to store them somewhere to reference them. +/// This can go away once https://github.com/projectfluent/fluent-rs/issues/103 lands. +pub struct Resources(Vec<(&'static str, FluentResource)>); + +impl Resources { + /// Parse a list of sources into a list of resources. + pub fn new(sources: Sources) -> Resources { + Resources( + sources + .iter() + .map(|(locale, source)| { + ( + *locale, + FluentResource::try_new(source.to_string()) + .expect("baked .ftl translation failed to parse"), + ) + }) + .collect(), + ) + } +} + +/// Sources; an array mapping &'static strs to fluent source strings. Instantiated only by the `impl_localize!` macro. +pub type Sources = &'static [(&'static str, &'static str)]; + +pub use fluent_bundle::FluentValue as I18nValue; + #[cfg(test)] mod tests { use super::*; const SOURCES: Sources = &[ ( - Locale("en_US"), + "en_US", r#" greeting = Hello, { $name }! You are { $hours } hours old. goodbye = Goodbye. "#, ), ( - Locale("en_AU"), + "en_AU", r#" greeting = G'day, { $name }! You are { $hours } hours old. goodbye = Hooroo. "#, ), ( - Locale("es_MX"), + "es_MX", r#" greeting = ¡Hola, { $name }! Tienes { $hours } horas. goodbye = Adiós. "#, ), ( - Locale("de_DE"), + "de_DE", "greeting = Hallo { $name }! Du bist { $hours } Stunden alt.", ), ]; @@ -252,35 +233,31 @@ goodbye = Adiós. #[test] fn basic() -> Result<()> { let resources = Resources::new(SOURCES); - let bundles = StaticParser::new(&resources, Locale("en_US")); + let bundles = StaticParser::new(&resources, "en_US"); let name = FluentValue::from("Jamie"); let hours = FluentValue::from(190321.31); let args = &[("name", &name), ("hours", &hours)][..]; assert_eq!( - bundles.localize(&[Locale("en_US")], "greeting", args)?, + bundles.localize(&["en_US"], "greeting", args)?, "Hello, Jamie! You are 190321.31 hours old." ); assert_eq!( - bundles.localize(&[Locale("es_MX")], "greeting", args)?, + bundles.localize(&["es_MX"], "greeting", args)?, "¡Hola, Jamie! Tienes 190321.31 horas." ); assert_eq!( - bundles.localize(&[Locale("de_DE")], "greeting", args)?, + bundles.localize(&["de_DE"], "greeting", args)?, "Hallo Jamie! Du bist 190321.31 Stunden alt." ); // missing messages should fall back to first available assert_eq!( - bundles.localize( - &[Locale("de_DE"), Locale("es_MX"), Locale("en_US")], - "goodbye", - &[] - )?, + bundles.localize(&["de_DE", "es_MX", "en_US"], "goodbye", &[])?, "Adiós." ); - if let Ok(_) = bundles.localize(&[Locale("en_US")], "bananas", &[]) { + if let Ok(_) = bundles.localize(&["en_US"], "bananas", &[]) { panic!("Should return Err on missing message"); } @@ -290,33 +267,33 @@ goodbye = Adiós. #[test] fn create_locale_chain() { let resources = Resources::new(SOURCES); - let bundles = StaticParser::new(&resources, Locale("en_US")); + let bundles = StaticParser::new(&resources, "en_US"); // accept-language parser works + short-code lookup works assert_eq!( bundles.create_locale_chain(&[], Some("en_US, es_MX; q=0.5")), - &[Locale("en_US"), Locale("en_AU"), Locale("es_MX")] + &["en_US", "en_AU", "es_MX"] ); // first choice has precedence assert_eq!( bundles.create_locale_chain(&["es_MX"], Some("en_US; q=0.5")), - &[Locale("es_MX"), Locale("en_US"), Locale("en_AU")] + &["es_MX", "en_US", "en_AU"] ); // short codes work assert_eq!( bundles.create_locale_chain(&[], Some("en")), - &[Locale("en_AU"), Locale("en_US")] + &["en_US", "en_AU"] ); // default works - assert_eq!(bundles.create_locale_chain(&[], None), &[Locale("en_US")]); + assert_eq!(bundles.create_locale_chain(&[], None), &["en_US"]); // missing languages fall through to default assert_eq!( bundles.create_locale_chain(&["zh_HK"], Some("xy_ZW")), - &[Locale("en_US")] + &["en_US"] ); } } diff --git a/askama_shared/src/lib.rs b/askama_shared/src/lib.rs index a5e9810ce..81bae58eb 100644 --- a/askama_shared/src/lib.rs +++ b/askama_shared/src/lib.rs @@ -2,10 +2,10 @@ #[macro_use] extern crate serde_derive; -#[cfg(feature = "with-i18n")] -extern crate accept_language; #[cfg(feature = "with-i18n")] extern crate fluent_bundle; +#[cfg(feature = "with-i18n")] +extern crate fluent_locale; use toml; From 08d13e6aa677c03ff127623f31b7c7ed913580a5 Mon Sep 17 00:00:00 2001 From: James Gilles Date: Mon, 6 May 2019 20:04:57 -0400 Subject: [PATCH 3/3] Switch to using multiple separate FluentResources for every input file, forgive some errors --- askama_shared/src/i18n_runtime.rs | 63 +++++++++++++++---------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/askama_shared/src/i18n_runtime.rs b/askama_shared/src/i18n_runtime.rs index a34cbfd96..bd85a0dd0 100644 --- a/askama_shared/src/i18n_runtime.rs +++ b/askama_shared/src/i18n_runtime.rs @@ -46,16 +46,18 @@ impl<'a> StaticParser<'a> { let mut bundles = HashMap::new(); let mut available = Vec::new(); - for (locale, resource) in resources.0.iter() { + for (locale, resources) in resources.0.iter() { // confusingly, this value is used by fluent for number and date formatting only. // we have to implement looking up missing messages in other bundles ourselves. let fallback_chain = &[locale]; let mut bundle = FluentBundle::new(fallback_chain); - bundle - .add_resource(resource) - .expect("failed to add resource"); + for resource in resources { + bundle + .add_resource(resource) + .expect("failed to add resource"); + } bundles.insert(*locale, bundle); available.push(*locale); @@ -129,32 +131,24 @@ impl<'a> StaticParser<'a> { let args = args.as_ref(); for locale in locale_chain { - let bundle = self.bundles.get(locale); - let bundle = if let Some(bundle) = bundle { - bundle - } else { - // TODO warn? - continue; - }; + let bundle = self + .bundles + .get(locale) + .expect("invariant violated: available locales should have matching bundles"); // this API is weirdly awful; // format returns Option<(String, Vec)> // which we have to cope with let result = bundle.format(message, args); - if let Some((result, errs)) = result { - if errs.len() == 0 { - return Ok(result); - } else { - continue; + if let Some((result, _errs)) = result { + return Ok(result); - // TODO: fluent degrades gracefully; maybe just warn here? - // Err(Error::I18n(errs.pop().unwrap())) - } + // TODO: warn on errors here? } } // nowhere to fall back to Err(Error::NoTranslationsForMessage(format!( - "no translations for message {} in locale chain {:?}", + "no non-erroring translations for message {} in locale chain {:?}", message, locale_chain ))) } @@ -173,7 +167,7 @@ impl<'a> StaticParser<'a> { /// because FluentBundle can only take FluentResources by reference; /// we have to store them somewhere to reference them. /// This can go away once https://github.com/projectfluent/fluent-rs/issues/103 lands. -pub struct Resources(Vec<(&'static str, FluentResource)>); +pub struct Resources(Vec<(&'static str, Vec)>); impl Resources { /// Parse a list of sources into a list of resources. @@ -181,11 +175,16 @@ impl Resources { Resources( sources .iter() - .map(|(locale, source)| { + .map(|(locale, sources)| { ( *locale, - FluentResource::try_new(source.to_string()) - .expect("baked .ftl translation failed to parse"), + sources + .iter() + .map(|source| { + FluentResource::try_new(source.to_string()) + .expect("baked .ftl translation failed to parse") + }) + .collect(), ) }) .collect(), @@ -194,7 +193,7 @@ impl Resources { } /// Sources; an array mapping &'static strs to fluent source strings. Instantiated only by the `impl_localize!` macro. -pub type Sources = &'static [(&'static str, &'static str)]; +pub type Sources = &'static [(&'static str, &'static [&'static str])]; pub use fluent_bundle::FluentValue as I18nValue; @@ -205,28 +204,28 @@ mod tests { const SOURCES: Sources = &[ ( "en_US", - r#" + &[r#" greeting = Hello, { $name }! You are { $hours } hours old. goodbye = Goodbye. -"#, +"#], ), ( "en_AU", - r#" + &[r#" greeting = G'day, { $name }! You are { $hours } hours old. goodbye = Hooroo. -"#, +"#], ), ( "es_MX", - r#" + &[r#" greeting = ¡Hola, { $name }! Tienes { $hours } horas. goodbye = Adiós. -"#, +"#], ), ( "de_DE", - "greeting = Hallo { $name }! Du bist { $hours } Stunden alt.", + &["greeting = Hallo { $name }! Du bist { $hours } Stunden alt."], ), ];