Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for NFKD and the decomposed counterpart of UTS 46 without ignored and disallowed #1967

Merged
merged 10 commits into from
Jun 2, 2022
Merged

Add support for NFKD and the decomposed counterpart of UTS 46 without ignored and disallowed #1967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
f79ea99
Add support for NFKD and the decomposed counterpart of NFKC_CaseFold …
hsivonen May 30, 2022
f1ad72d
Consolidate complex decomposition data
hsivonen May 31, 2022
b08ad82
Avoid a clone that allocates instead of just copying a pointer
hsivonen May 31, 2022
e281e9f
Merge branch 'main' into normalizerdata
hsivonen Jun 1, 2022
dd1d56b
Document normalizer data trade-offs
hsivonen Jun 1, 2022
2b7d67a
Sync normalizer README with lib.rs
hsivonen Jun 1, 2022
30b4b4a
Extract the right ICU4C data for UTS 46
hsivonen Jun 2, 2022
97129a9
Flatten out a level of reference indirection on decomposition_starts_…
hsivonen Jun 1, 2022
eae8a1d
Merge branch 'main' into normalizerdata
hsivonen Jun 2, 2022
676133b
Use 3 bytes per supplementary-plane character in normalization data
hsivonen Jun 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions experimental/collator/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,12 @@ icu_provider = { version = "0.6", path = "../../provider/core", features = ["mac
icu_locid = { version = "0.6", path = "../../components/locid" }
icu_normalizer = { version = "0.6", path = "../../experimental/normalizer" }
icu_properties = { version = "0.6", path = "../../components/properties" }
icu_uniset = { version = "0.5", path = "../../utils/uniset" }
serde = { version = "1.0", default-features = false, features = ["derive", "alloc"], optional = true }
zerovec = { version = "0.7", path = "../../utils/zerovec", features = ["serde"] }
utf8_iter = "1.0"
crabbake = { version = "0.4", path = "../../experimental/crabbake", optional = true, features = ["derive"] }
zerofrom = { version = "0.1.0", path = "../../utils/zerofrom" }

[dev-dependencies]
icu_testdata = { version = "0.6", path = "../../provider/testdata" }
Expand All @@ -56,5 +58,5 @@ bench = false # This option is required for Benchmark CI

[features]
default = []
serde = ["dep:serde", "zerovec/serde", "icu_char16trie/serde", "icu_properties/serde", "icu_normalizer/serde", "icu_codepointtrie/serde"]
datagen = ["serde", "crabbake", "zerovec/crabbake", "icu_char16trie/crabbake", "icu_properties/crabbake", "icu_normalizer/crabbake", "icu_codepointtrie/crabbake"]
serde = ["dep:serde", "zerovec/serde", "icu_char16trie/serde", "icu_properties/serde", "icu_normalizer/serde", "icu_uniset/serde", "icu_codepointtrie/serde"]
datagen = ["serde", "crabbake", "zerovec/crabbake", "icu_char16trie/crabbake", "icu_properties/crabbake", "icu_normalizer/crabbake", "icu_uniset/crabbake", "icu_codepointtrie/crabbake"]
22 changes: 22 additions & 0 deletions experimental/collator/src/comparison.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ use core::cmp::Ordering;
use core::convert::TryFrom;
use icu_locid::Locale;
use icu_normalizer::provider::CanonicalDecompositionDataV1Marker;
use icu_normalizer::provider::CanonicalDecompositionTablesV1Marker;
use icu_normalizer::Decomposition;
use icu_properties::provider::CanonicalCombiningClassV1Marker;
use icu_provider::DataPayload;
Expand Down Expand Up @@ -72,6 +73,7 @@ pub struct Collator {
options: CollatorOptions,
reordering: Option<DataPayload<CollationReorderingV1Marker>>,
decompositions: DataPayload<CanonicalDecompositionDataV1Marker>,
tables: DataPayload<CanonicalDecompositionTablesV1Marker>,
ccc: DataPayload<CanonicalCombiningClassV1Marker>,
lithuanian_dot_above: bool,
}
Expand All @@ -90,6 +92,7 @@ impl Collator {
+ ResourceProvider<CollationMetadataV1Marker>
+ ResourceProvider<CollationReorderingV1Marker>
+ ResourceProvider<CanonicalDecompositionDataV1Marker>
+ ResourceProvider<CanonicalDecompositionTablesV1Marker>
+ ResourceProvider<CanonicalCombiningClassV1Marker>
+ ?Sized,
{
Expand Down Expand Up @@ -203,6 +206,10 @@ impl Collator {
.load_resource(&DataRequest::default())?
.take_payload()?;

let tables: DataPayload<CanonicalDecompositionTablesV1Marker> = data_provider
.load_resource(&DataRequest::default())?
.take_payload()?;

let ccc: DataPayload<CanonicalCombiningClassV1Marker> =
icu_properties::maps::get_canonical_combining_class(data_provider)?;

Expand Down Expand Up @@ -246,6 +253,7 @@ impl Collator {
options: merged_options,
reordering,
decompositions,
tables,
ccc,
lithuanian_dot_above: metadata.lithuanian_dot_above(),
})
Expand All @@ -272,11 +280,15 @@ impl Collator {
return Decomposition::new(
decode_utf16(left.iter().copied()).map(utf16_error_to_replacement),
self.decompositions.get(),
self.tables.get(),
None,
&self.ccc.get().code_point_trie,
)
.cmp(Decomposition::new(
decode_utf16(right.iter().copied()).map(utf16_error_to_replacement),
self.decompositions.get(),
self.tables.get(),
None,
&self.ccc.get().code_point_trie,
));
}
Expand All @@ -301,11 +313,15 @@ impl Collator {
return Decomposition::new(
left.chars(),
self.decompositions.get(),
self.tables.get(),
None,
&self.ccc.get().code_point_trie,
)
.cmp(Decomposition::new(
right.chars(),
self.decompositions.get(),
self.tables.get(),
None,
&self.ccc.get().code_point_trie,
));
}
Expand All @@ -330,11 +346,15 @@ impl Collator {
return Decomposition::new(
left.chars(),
self.decompositions.get(),
self.tables.get(),
None,
&self.ccc.get().code_point_trie,
)
.cmp(Decomposition::new(
right.chars(),
self.decompositions.get(),
self.tables.get(),
None,
&self.ccc.get().code_point_trie,
));
}
Expand Down Expand Up @@ -411,6 +431,7 @@ impl Collator {
)
.unwrap(), // length already validated
self.decompositions.get(),
self.tables.get(),
&self.ccc.get().code_point_trie,
numeric_primary,
self.lithuanian_dot_above,
Expand All @@ -426,6 +447,7 @@ impl Collator {
)
.unwrap(), // length already validated
self.decompositions.get(),
self.tables.get(),
&self.ccc.get().code_point_trie,
numeric_primary,
self.lithuanian_dot_above,
Expand Down
Loading