Data and provider for compact decimal formatting

Cargo.toml

@@ -21,6 +21,7 @@ members = [
     "components/timezone",
     "experimental/bies",
     "experimental/casemapping",
+    "experimental/compactdecimal",
     "experimental/displaynames",
     "experimental/ixdtf",
     "experimental/relativetime",

experimental/compactdecimal/Cargo.toml

@@ -0,0 +1,44 @@
+# This file is part of ICU4X. For terms of use, please see the file
+# called LICENSE at the top level of the ICU4X source tree
+# (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+[package]
+name = "icu_compactdecimal"
+version = "0.1.0"
+authors = ["The ICU4X Project Developers"]
+edition = "2018"
+repository = "https://github.com/unicode-org/icu4x"
+license-file = "LICENSE"
+categories = ["internationalization"]
+# Keep this in sync with other crates unless there are exceptions
+include = [
+    "data/*",
+    "src/**/*",
+    "examples/**/*",
+    "benches/**/*",
+    "tests/**/*",
+    "Cargo.toml",
+    "LICENSE",
+    "README.md"
+]
+
+[dependencies]
+displaydoc = { version = "0.2.3", default-features = false }
+icu_decimal = { version = "1.0.0", path = "../../components/decimal" }
+icu_provider = { version = "1.0.0", path = "../../provider/core", features = ["macros"] }
+icu_plurals = { version = "1.0.0", path = "../../components/plurals" }
+zerovec = { version = "0.9", path = "../../utils/zerovec", features = ["yoke"] }
+serde = { version = "1.0", default-features = false, features = ["derive", "alloc"], optional = true }
+writeable = { version = "0.5", path = "../../utils/writeable" }
+fixed_decimal = { version = "0.5", path = "../../utils/fixed_decimal" }
+databake = { version = "0.1.0", path = "../../utils/databake", optional = true, features = ["derive"]}
+
+[dev-dependencies]
+icu_testdata = { path = "../../provider/testdata", default-features = false, features = ["icu_plurals", "icu_decimal"] }
+icu_locid = { version = "1.0.0", path = "../../components/locid" }
+
+[features]
+default = []
+std = ["icu_provider/std"]
+serde = ["dep:serde", "zerovec/serde", "icu_decimal/serde", "icu_plurals/serde"]
+datagen = ["std", "serde", "dep:databake", "zerovec/databake"]

experimental/compactdecimal/LICENSE

@@ -0,0 +1,51 @@
+UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+
+See Terms of Use <https://www.unicode.org/copyright.html>
+for definitions of Unicode Inc.’s Data Files and Software.
+
+NOTICE TO USER: Carefully read the following legal agreement.
+BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
+DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
+YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT.
+IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
+THE DATA FILES OR SOFTWARE.
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright © 1991-2022 Unicode, Inc. All rights reserved.
+Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Unicode data files and any associated documentation
+(the "Data Files") or Unicode software and any associated documentation
+(the "Software") to deal in the Data Files or Software
+without restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, and/or sell copies of
+the Data Files or Software, and to permit persons to whom the Data Files
+or Software are furnished to do so, provided that either
+(a) this copyright and permission notice appear with all copies
+of the Data Files or Software, or
+(b) this copyright and permission notice appear in associated
+Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder
+shall not be used in advertising or otherwise to promote the sale,
+use or other dealings in these Data Files or Software without prior
+written authorization of the copyright holder.
+
+—
+
+Portions of ICU4X may have been adapted from ICU4C and/or ICU4J.
+ICU 1.8.1 to ICU 57.1 © 1995-2016 International Business Machines Corporation and others.

experimental/compactdecimal/README.md

@@ -0,0 +1,17 @@
+# icu_compactdecimal [![crates.io](https://img.shields.io/crates/v/icu_compactdecimal)](https://crates.io/crates/icu_compactdecimal)
+
+🚧 \[Experimental\] Relative time formatting
+
+This module is published as its own crate ([`icu_compactdecimal`](https://docs.rs/icu_compactdecimal/latest/icu_compactdecimal/))
+and as part of the [`icu`](https://docs.rs/icu/latest/icu/) crate. See the latter for more details on the ICU4X project.
+
+<div class="stab unstable">
+🚧 This code is experimental; it may change at any time, in breaking or non-breaking ways,
+including in SemVer minor releases. It can be enabled with the "experimental" feature
+of the icu meta-crate. Use with caution.
+</div>
+
+
+## More Information
+
+For more information on development, authorship, contributing etc. please visit [`ICU4X home page`](https://github.com/unicode-org/icu4x).

experimental/compactdecimal/src/error.rs

@@ -0,0 +1,41 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use displaydoc::Display;
+use icu_decimal::DecimalError;
+use icu_plurals::PluralsError;
+use icu_provider::DataError;
+
+/// A list of error outcomes for various operations in the `icu_compactdecimal` crate.
+#[derive(Display, Debug, Copy, Clone, PartialEq)]
+#[non_exhaustive]
+pub enum CompactDecimalError {
+    /// An error originating from [`PluralRules`](icu_plurals::PluralRules).
+    #[displaydoc("Error loading plural rules: {0}")]
+    PluralRules(PluralsError),
+    /// An error originating from [`DataProvider`](icu_provider::DataProvider).
+    #[displaydoc("Error loading data: {0}")]
+    Data(DataError),
+    /// An error originating from [`FixedDecimalFormatter`](icu_decimal::FixedDecimalFormatter).
+    #[displaydoc("Error loading FixedDecimalFormatter: {0}")]
+    Decimal(DecimalError),
+}
+
+impl From<PluralsError> for CompactDecimalError {
+    fn from(e: PluralsError) -> Self {
+        CompactDecimalError::PluralRules(e)
+    }
+}
+
+impl From<DataError> for CompactDecimalError {
+    fn from(e: DataError) -> Self {
+        CompactDecimalError::Data(e)
+    }
+}
+
+impl From<DecimalError> for CompactDecimalError {
+    fn from(e: DecimalError) -> Self {
+        CompactDecimalError::Decimal(e)
+    }
+}

experimental/compactdecimal/src/lib.rs

@@ -0,0 +1,38 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+//! 🚧 \[Experimental\] Relative time formatting
+//!
+//! This module is published as its own crate ([`icu_compactdecimal`](https://docs.rs/icu_compactdecimal/latest/icu_compactdecimal/))
+//! and as part of the [`icu`](https://docs.rs/icu/latest/icu/) crate. See the latter for more details on the ICU4X project.
+//!
+//! <div class="stab unstable">
+//! 🚧 This code is experimental; it may change at any time, in breaking or non-breaking ways,
+//! including in SemVer minor releases. It can be enabled with the "experimental" feature
+//! of the icu meta-crate. Use with caution.
+//! </div>
+//!
+
+// https://github.com/unicode-org/icu4x/blob/main/docs/process/boilerplate.md#library-annotations
+#![cfg_attr(not(test), no_std)]
+#![cfg_attr(
+    not(test),
+    deny(
+        clippy::indexing_slicing,
+        clippy::unwrap_used,
+        clippy::expect_used,
+        clippy::panic,
+        clippy::exhaustive_structs,
+        clippy::exhaustive_enums,
+        missing_debug_implementations,
+    )
+)]
+#![warn(missing_docs)]
+
+extern crate alloc;
+
+mod error;
+pub mod provider;
+
+pub use error::CompactDecimalError;

experimental/compactdecimal/src/provider.rs

@@ -0,0 +1,127 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+// Provider structs must be stable.
+#![allow(clippy::exhaustive_structs, clippy::exhaustive_enums)]
+// Suppress a warning on zerovec::makevarule.
+#![allow(missing_docs)]
+
+//! Data provider struct definitions for this ICU4X component.
+//!
+//! Read more about data providers: [`icu_provider`]
+
+use alloc::borrow::Cow;
+use icu_provider::{yoke, zerofrom};
+use zerovec::ZeroMap2d;
+
+/// Relative time format V1 data struct.
+/// As in CLDR, this is a mapping from type (a power of ten, corresponding to
+/// the magnitude of the number being formatted) and count (a plural case or an
+/// explicit 1) to a pattern.
+///
+/// However, plural cases that are identical to the other case are omitted, thus
+/// given
+/// > (1000, one) ↦ 0K, (1000, other) ↦ 0K
+///
+/// only
+/// > (1000, other) ↦ 0K
+///
+/// is stored.
+///
+/// Further, if all plural cases are compatible across consecutive types, the
+/// larger types are omitted, thus given
+/// > (1000, other) ↦ 0K, (10000, other) ↦ 00K, (100000, other) ↦ 000K
+///
+/// only
+/// > (1000, other) ↦ 0K
+///
+/// is stored.
+///
+/// Finally, the pattern indicating noncompact notation for the first few powers
+/// of ten is omitted; that is, there is an implict (1, other) ↦ 0.
+#[icu_provider::data_struct(
+    LongCompactDecimalFormatDataV1Marker = "compactdecimal/long@1",
+    ShortCompactDecimalFormatDataV1Marker = "compactdecimal/short@1"
+)]
+#[derive(Debug, Clone, Default, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
+#[cfg_attr(
+    feature = "datagen", 
+    derive(serde::Serialize, databake::Bake),
+    databake(path = icu_compactdecimal::provider)
+)]
+#[yoke(prove_covariance_manually)]
+pub struct CompactDecimalPatternDataV1<'data> {
+    /// A map keyed on log10 of the CLDR `type` attribute and the CLDR `count` attribute.
+    #[cfg_attr(feature = "serde", serde(borrow))]
+    pub patterns: ZeroMap2d<'data, i8, Count, PatternULE>,
+}
+
+/// A CLDR plural keyword, or the explicit value 1.
+/// See <https://www.unicode.org/reports/tr35/tr35-numbers.html#Language_Plural_Rules>.
+#[zerovec::make_ule(CountULE)]
+#[zerovec::derive(Debug)]
+#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
+#[cfg_attr(
+    feature = "datagen", 
+    derive(serde::Serialize, databake::Bake),
+    databake(path = icu_compactdecimal::provider)
+)]
+#[repr(u8)]
+pub enum Count {
+    /// The CLDR keyword `zero`.
+    Zero = 0,
+    /// The CLDR keyword `one`.
+    One = 1,
+    /// The CLDR keyword `two`.
+    Two = 2,
+    /// The CLDR keyword `few`.
+    Few = 3,
+    /// The CLDR keyword `many`.
+    Many = 4,
+    /// The CLDR keyword `other`.
+    Other = 5,
+    /// The explicit 1 case, see <https://www.unicode.org/reports/tr35/tr35-numbers.html#Explicit_0_1_rules>.
+    Explicit1 = 6,
+    // NOTE(egg): No explicit 0, because the compact decimal pattern selection
+    // algorithm does not allow such a thing to arise.
+}
+
+/// A compact decimal pattern, representing some literal text with an optional
+/// placeholder, and the power of 10 expressed by the text.
+#[derive(
+    Debug, Clone, Default, PartialEq, yoke::Yokeable, zerofrom::ZeroFrom, Ord, PartialOrd, Eq,
+)]
+#[zerovec::make_varule(PatternULE)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
+#[cfg_attr(
+    feature = "datagen", 
+    derive(serde::Serialize, databake::Bake),
+    databake(path = icu_compactdecimal::provider),
+    zerovec::derive(Serialize),
+)]
+#[zerovec::derive(Debug)]
+#[cfg_attr(feature = "serde", zerovec::derive(Deserialize))]
+pub struct Pattern<'data> {
+    /// The compact decimal exponent, e.g., 6 for "million".
+    /// The value 0 indicates that compact notation is not used; in that case,
+    /// literal text must be empty; this corresponds to the CLDR pattern "0".
+    /// This is derived from the numbers of 0s in the pattern and the associated
+    /// `type` attribute; it is a more convenient representation than the number
+    /// of 0s, because it is often common to multiple types; for instance, the
+    /// following correspond to the same [`Pattern`]:
+    ///   <pattern type="1000000" count="other">0 M</pattern>
+    ///   <pattern type="10000000" count="other">00 M</pattern>
+    pub exponent: i8,
+    /// The index in literal_text before which the placeholder is inserted;
+    /// this is 0 for insertion at the beginning, which is most common.
+    /// The value 255 indicates that the pattern does not have a placeholder,
+    /// as in French "mille" for 1000.
+    pub index: u8,
+    #[cfg_attr(feature = "serde", serde(borrow))]
+    /// The underlying CLDR pattern with the placeholder removed, e.g.,
+    /// " M" for the pattern "000 M"
+    pub literal_text: Cow<'data, str>,
+}

provider/datagen/Cargo.toml

@@ -51,6 +51,7 @@ icu_casemapping = { version = "0.7.0", path = "../../experimental/casemapping",
 icu_segmenter = { version = "0.7.0", path = "../../experimental/segmenter", features = ["datagen", "lstm"], optional = true }
 icu_displaynames = { version = "0.7.0", path = "../../experimental/displaynames", features = ["datagen"], optional = true }
 icu_relativetime = { version = "0.1.0", path = "../../experimental/relativetime", features = ["datagen"], optional = true }
+icu_compactdecimal = { version = "0.1.0", path = "../../experimental/compactdecimal", features = ["datagen"], optional = true }
 
 # ICU provider infrastructure
 icu_provider = { version = "1.0.0", path = "../core", features = ["std", "log_error_context", "datagen"]}
@@ -96,7 +97,7 @@ icu = { path = "../../components/icu" }
 
 [features]
 default = []
-experimental = ["icu_casemapping", "icu_segmenter", "icu_datetime/experimental", "icu_displaynames", "icu_relativetime"]
+experimental = ["icu_casemapping", "icu_segmenter", "icu_datetime/experimental", "icu_displaynames", "icu_relativetime", "icu_compactdecimal"]
 bin = ["clap", "eyre", "simple_logger"]
 
 [[bin]]

provider/datagen/src/registry.rs

@@ -21,6 +21,8 @@ use icu_timezone::provider::*;
 #[cfg(feature = "experimental")]
 use icu_casemapping::provider::*;
 #[cfg(feature = "experimental")]
+use icu_compactdecimal::provider::*;
+#[cfg(feature = "experimental")]
 use icu_displaynames::provider::*;
 #[cfg(feature = "experimental")]
 use icu_relativetime::provider::*;
@@ -252,6 +254,8 @@ registry!(
         LongYearRelativeTimeFormatDataV1Marker,
         ShortYearRelativeTimeFormatDataV1Marker,
         NarrowYearRelativeTimeFormatDataV1Marker,
+        LongCompactDecimalFormatDataV1Marker,
+        ShortCompactDecimalFormatDataV1Marker,
     }
 );