Skip to content

Commit

Permalink
Rollup merge of rust-lang#104047 - crlf0710:icu_based_list_formatting…
Browse files Browse the repository at this point in the history
…, r=davidtwco

Diagnostics `icu4x` based list formatting.

This adds a new kind of `DiagnosticArg` and add the ability to convert it to a `FluentValue::Custom`. When emitting fluent output, it makes use of `ListFormatter` from `icu4x` project to convert it to localized version of list following the fluent locale, as a kind of eager translation.

Tested locally with locales like `en`, `ja`, etc, and they work fine. <del>Though neither `zh-CN` nor `zh-Hans` works correctly, it seems they fallback to `und` locale somehow, emitting only comma-based list. I believe this is an internal issue of `icu4x` itself.</del>(Works fine after rust-lang#104047 (comment))

Would love to hear what others think.

r? `@davidtwco`
cc `@Manishearth`
  • Loading branch information
Manishearth authored Nov 18, 2022
2 parents 774f17f + d75c76d commit 2709c2d
Show file tree
Hide file tree
Showing 19 changed files with 2,644 additions and 23 deletions.
178 changes: 176 additions & 2 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static",
"memchr",
"regex-automata",
"regex-automata 0.1.10",
]

[[package]]
Expand Down Expand Up @@ -1707,6 +1707,73 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a"

[[package]]
name = "icu_list"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c40218275f081c4493f190357c5395647b06734c2dc3dcb41cc099a0f60168b1"
dependencies = [
"displaydoc",
"icu_locid",
"icu_provider",
"regex-automata 0.2.0",
"writeable",
"zerovec",
]

[[package]]
name = "icu_locid"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34b3de5d99a0e275fe6193b9586dbf37364daebc0d39c89b5cf8376a53b789e8"
dependencies = [
"displaydoc",
"litemap",
"tinystr",
"writeable",
"zerovec",
]

[[package]]
name = "icu_provider"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f911086e3c521a8a824d4f8bfd87769645ced2f07ff913b521c0d793be07100"
dependencies = [
"displaydoc",
"icu_locid",
"icu_provider_macros",
"stable_deref_trait",
"writeable",
"yoke",
"zerofrom",
"zerovec",
]

[[package]]
name = "icu_provider_adapters"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "980c71d8a91b246ebbb97847178a4b816eea39d1d550c70ee566384555bb6545"
dependencies = [
"icu_locid",
"icu_provider",
"tinystr",
"yoke",
"zerovec",
]

[[package]]
name = "icu_provider_macros"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38cf6f5b65cf81f0b4298da647101acbfe6ae0e25263f92bd7a22597e9d6d606"
dependencies = [
"proc-macro2",
"quote",
"syn",
]

[[package]]
name = "idna"
version = "0.2.0"
Expand Down Expand Up @@ -2034,6 +2101,12 @@ dependencies = [
"walkdir",
]

[[package]]
name = "litemap"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f34a3f4798fac63fb48cf277eefa38f94d3443baff555bb98e4f56bc9092368e"

[[package]]
name = "lld-wrapper"
version = "0.1.0"
Expand Down Expand Up @@ -2100,7 +2173,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
dependencies = [
"regex-automata",
"regex-automata 0.1.10",
]

[[package]]
Expand Down Expand Up @@ -2927,6 +3000,15 @@ dependencies = [
"regex-syntax",
]

[[package]]
name = "regex-automata"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9368763f5a9b804326f3af749e16f9abf378d227bcdee7634b13d8f17793782"
dependencies = [
"memchr",
]

[[package]]
name = "regex-syntax"
version = "0.6.26"
Expand Down Expand Up @@ -3202,6 +3284,18 @@ dependencies = [
"rustc_span",
]

[[package]]
name = "rustc_baked_icu_data"
version = "0.0.0"
dependencies = [
"icu_list",
"icu_locid",
"icu_provider",
"icu_provider_adapters",
"litemap",
"zerovec",
]

[[package]]
name = "rustc_borrowck"
version = "0.0.0"
Expand Down Expand Up @@ -3422,13 +3516,18 @@ version = "0.0.0"
dependencies = [
"fluent-bundle",
"fluent-syntax",
"icu_list",
"icu_locid",
"icu_provider_adapters",
"intl-memoizer",
"rustc_baked_icu_data",
"rustc_data_structures",
"rustc_macros",
"rustc_serialize",
"rustc_span",
"tracing",
"unic-langid",
"writeable",
]

[[package]]
Expand Down Expand Up @@ -4934,6 +5033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aeafdfd935e4a7fe16a91ab711fa52d54df84f9c8f7ca5837a9d1d902ef4c2"
dependencies = [
"displaydoc",
"zerovec",
]

[[package]]
Expand Down Expand Up @@ -5546,6 +5646,12 @@ version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"

[[package]]
name = "writeable"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8e6ab4f5da1b24daf2c590cfac801bacb27b15b4f050e84eb60149ea726f06b"

[[package]]
name = "xattr"
version = "0.2.2"
Expand Down Expand Up @@ -5598,3 +5704,71 @@ checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1"
dependencies = [
"winapi",
]

[[package]]
name = "yoke"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fe1d55ca72c32d573bfbd5cb2f0ca65a497854c44762957a6d3da96041a5184"
dependencies = [
"serde",
"stable_deref_trait",
"yoke-derive",
"zerofrom",
]

[[package]]
name = "yoke-derive"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1346e4cd025ae818b88566eac7eb65ab33a994ea55f355c86889af2e7e56b14e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]

[[package]]
name = "zerofrom"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79e9355fccf72b04b7deaa99ce7a0f6630530acf34045391b74460fcd714de54"
dependencies = [
"zerofrom-derive",
]

[[package]]
name = "zerofrom-derive"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e8aa86add9ddbd2409c1ed01e033cd457d79b1b1229b64922c25095c595e829"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]

[[package]]
name = "zerovec"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d919a74c17749ccb17beaf6405562e413cd94e98ba52ca1e64bbe7eefbd8b8"
dependencies = [
"yoke",
"zerofrom",
"zerovec-derive",
]

[[package]]
name = "zerovec-derive"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490e5f878c2856225e884c35927e7ea6db3c24cdb7229b72542c7526ad7ed49e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]
15 changes: 15 additions & 0 deletions compiler/rustc_baked_icu_data/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[package]
name = "rustc_baked_icu_data"
version = "0.0.0"
edition = "2021"

[dependencies]
icu_list = "1.0.0"
icu_locid = "1.0.0"
icu_provider = "1.0.1"
icu_provider_adapters = "1.0.0"
litemap = "0.6.0"
zerovec = "0.9.0"

[features]
rustc_use_parallel_compiler = ['icu_provider/sync']
42 changes: 42 additions & 0 deletions compiler/rustc_baked_icu_data/src/data/any.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// @generated
impl AnyProvider for BakedDataProvider {
fn load_any(&self, key: DataKey, req: DataRequest) -> Result<AnyResponse, DataError> {
const ANDLISTV1MARKER: ::icu_provider::DataKeyHash =
::icu_list::provider::AndListV1Marker::KEY.hashed();
const COLLATIONFALLBACKSUPPLEMENTV1MARKER: ::icu_provider::DataKeyHash =
::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY
.hashed();
const LOCALEFALLBACKLIKELYSUBTAGSV1MARKER: ::icu_provider::DataKeyHash =
::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY
.hashed();
const LOCALEFALLBACKPARENTSV1MARKER: ::icu_provider::DataKeyHash =
::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY
.hashed();
#[allow(clippy::match_single_binding)]
match key.hashed() {
ANDLISTV1MARKER => list::and_v1::DATA
.get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
.copied()
.map(AnyPayload::from_static_ref)
.ok_or(DataErrorKind::MissingLocale),
COLLATIONFALLBACKSUPPLEMENTV1MARKER => fallback::supplement::co_v1::DATA
.get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
.copied()
.map(AnyPayload::from_static_ref)
.ok_or(DataErrorKind::MissingLocale),
LOCALEFALLBACKLIKELYSUBTAGSV1MARKER => fallback::likelysubtags_v1::DATA
.get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
.copied()
.map(AnyPayload::from_static_ref)
.ok_or(DataErrorKind::MissingLocale),
LOCALEFALLBACKPARENTSV1MARKER => fallback::parents_v1::DATA
.get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
.copied()
.map(AnyPayload::from_static_ref)
.ok_or(DataErrorKind::MissingLocale),
_ => Err(DataErrorKind::MissingDataKey),
}
.map_err(|e| e.with_req(key, req))
.map(|payload| AnyResponse { payload: Some(payload), metadata: Default::default() })
}
}
Loading

0 comments on commit 2709c2d

Please sign in to comment.