diff --git a/common/properties/coverageLevels.txt b/common/properties/coverageLevels.txt index fc4a5d31df7..3a2bb78c703 100644 --- a/common/properties/coverageLevels.txt +++ b/common/properties/coverageLevels.txt @@ -144,7 +144,6 @@ th ; modern ; Thai ti ; basic ; Tigrinya tk ; modern ; Turkmen to ; basic ; Tongan -tok ; basic ; Toki Pona tr ; modern ; Turkish tt ; basic ; Tatar ug ; basic ; Uyghur diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/StandardCodes.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/StandardCodes.java index f3c27281717..02f8f9fe478 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/StandardCodes.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/StandardCodes.java @@ -149,6 +149,10 @@ private Map> getCodeData(CodeType type) { return type_code_data.get(type); } + public Set getCodes(CodeType type) { + return type_code_data.get(type).keySet(); + } + /** * Get at the language registry values, as a Map from label to value. * @@ -275,7 +279,7 @@ public Set getGoodAvailableCodes(CodeType type) { case script: return sd.getCLDRScriptCodes(); case tzid: - break; // nothing special + return sd.getCLDRTimezoneCodes(); default: for (Iterator it = result.iterator(); it.hasNext(); ) { String code = it.next(); diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java index b2bb5111875..062c0e8dfcc 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java @@ -4,10 +4,13 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import com.google.common.collect.TreeMultimap; import com.ibm.icu.impl.IterableComparator; import com.ibm.icu.impl.Relation; @@ -74,6 +77,7 @@ import org.unicode.cldr.util.GrammarInfo.GrammaticalScope; import org.unicode.cldr.util.GrammarInfo.GrammaticalTarget; import org.unicode.cldr.util.Rational.RationalParser; +import org.unicode.cldr.util.StandardCodes.CodeType; import org.unicode.cldr.util.StandardCodes.LstrType; import org.unicode.cldr.util.SupplementalDataInfo.BasicLanguageData.Type; import org.unicode.cldr.util.SupplementalDataInfo.NumberingSystemInfo.NumberingSystemType; @@ -981,6 +985,10 @@ public enum RBNFGroup { public Map, String> bcp47Since = new TreeMap<>(); public Map, String> bcp47Preferred = new TreeMap<>(); public Map, String> bcp47Deprecated = new TreeMap<>(); + + Map> bcp47KeyToSubtypeToInfo = new TreeMap<>(); + Map> bcp47KeyToAliasToSubtype = new TreeMap<>(); + public Map bcp47ValueType = new TreeMap<>(); public Map> validityInfo = new LinkedHashMap<>(); @@ -1145,6 +1153,34 @@ private SupplementalDataInfo(File directory) { this.validity = Validity.getInstance(directory.toString() + "/../validity/"); } // hide + public static class Bcp47KeyInfo { + public Bcp47KeyInfo( + Set aliases, + String description, + String since, + String preferred, + String deprecated) { + this.description = description; + this.deprecated = !(deprecated == null || deprecated.equals("false")); + this.preferred = preferred; + this.since = since == null ? null : VersionInfo.getInstance(since); + this.aliases = aliases; + } + + final String description; + final VersionInfo since; + final String preferred; + final boolean deprecated; + final Set aliases; + + @Override + public String toString() { + return String.format( + "{description=«%s» since=%s preferred=%s deprecated=%s aliases=%s}", + description, since, preferred, deprecated, aliases); + } + } + private void makeStuffSafe() { // now make stuff safe allLanguages.addAll(languageToPopulation.keySet()); @@ -1224,19 +1260,54 @@ private void makeStuffSafe() { } typeToLocaleToDayPeriodInfo = CldrUtility.protectCollection(typeToLocaleToDayPeriodInfo); languageMatch = CldrUtility.protectCollection(languageMatch); - bcp47Key2Subtypes.freeze(); + bcp47Extension2Keys.freeze(); - bcp47Aliases.freeze(); + bcp47Key2Subtypes.freeze(); + CldrUtility.protectCollection(bcp47ValueType); if (bcp47Key2Subtypes.isEmpty()) { throw new InternalError( "No BCP47 key 2 subtype data was loaded from bcp47 dir " + getBcp47Directory().getAbsolutePath()); } + + bcp47Aliases.freeze(); CldrUtility.protectCollection(bcp47Descriptions); CldrUtility.protectCollection(bcp47Since); CldrUtility.protectCollection(bcp47Preferred); CldrUtility.protectCollection(bcp47Deprecated); - CldrUtility.protectCollection(bcp47ValueType); + + // create clean structure + + for (Entry> entry : bcp47Extension2Keys.keyValuesSet()) { + for (String key : entry.getValue()) { + Map subtypeToInfo = bcp47KeyToSubtypeToInfo.get(key); + if (subtypeToInfo == null) { + bcp47KeyToSubtypeToInfo.put(key, subtypeToInfo = new TreeMap<>()); + } + Map aliasToRegular = bcp47KeyToAliasToSubtype.get(key); + if (aliasToRegular == null) { + bcp47KeyToAliasToSubtype.put(key, aliasToRegular = new TreeMap<>()); + } + for (String subtype : bcp47Key2Subtypes.get(key)) { + final R2 pair = R2.of(key, subtype); + final Set aliases = bcp47Aliases.get(pair); + final Bcp47KeyInfo info = + new Bcp47KeyInfo( + aliases, + bcp47Descriptions.get(pair), + bcp47Since.get(pair), + bcp47Preferred.get(pair), + bcp47Deprecated.get(pair)); + subtypeToInfo.put(subtype, info); + final Map aliasToRegularFinal = aliasToRegular; + if (aliases != null) { + aliases.forEach(x -> aliasToRegularFinal.put(x, subtype)); + } + } + } + } + bcp47KeyToSubtypeToInfo = CldrUtility.protectCollection(bcp47KeyToSubtypeToInfo); + bcp47KeyToAliasToSubtype = CldrUtility.protectCollection(bcp47KeyToAliasToSubtype); CoverageLevelInfo.fixEU(coverageLevels, this); coverageLevels = Collections.unmodifiableSortedSet(coverageLevels); @@ -5173,4 +5244,64 @@ public UnitPrefixInfo getUnitPrefixInfo(String prefix) { public Set getUnitPrefixes() { return unitPrefixInfo.keySet(); } + + /** + * Filter out deprecated items. This is more complicated than it seems. The deprecation is in + * timezones.xml, eg: We need to find the + * short id's that are deprecated, put there is a problem due to + * https://unicode-org.atlassian.net/browse/CLDR-17412. + * + *

America/Nipigon, America/Thunder_Bay, America/Rainy_River + */ + Supplier> goodTimezones = + Suppliers.memoize( + new Supplier>() { + + @Override + public Set get() { + Set availableLongTz = sc.getAvailableCodes(CodeType.tzid); + Set result = null; + if (true) { // hack for now + final Set hack = + Set.of( + "America/Santa_Isabel", + "Australia/Currie", + "America/Yellowknife", + "America/Rainy_River", + "America/Thunder_Bay", + "America/Nipigon", + "America/Pangnirtung", + "Europe/Uzhgorod", + "Europe/Zaporozhye", + "Pacific/Johnston"); + result = Set.copyOf(Sets.difference(availableLongTz, hack)); + } else { // TODO restore when CLDR-17412 is fixed + Map aliasToRegular = + bcp47KeyToAliasToSubtype.get("tz"); + Map subtypeToInfo = + bcp47KeyToSubtypeToInfo.get("tz"); + result = + availableLongTz.stream() + .filter( + x -> { + String shortId = aliasToRegular.get(x); + Bcp47KeyInfo info = + subtypeToInfo.get(shortId); + System.out.println( + String.format( + "%s %s %s", + x, shortId, info)); + return !info.deprecated; + }) + .collect(Collectors.toUnmodifiableSet()); + } + return result; + } + }); + + public Set getCLDRTimezoneCodes() { + return goodTimezones.get(); + } } diff --git a/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/Locales.txt b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/Locales.txt index 26646a2211a..0422bacff8a 100644 --- a/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/Locales.txt +++ b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/Locales.txt @@ -207,7 +207,7 @@ Google ; nn ; Modern ; Nynorsk Google ; no ; modern ; T2 Norwegian (Bokmål) Google ; or ; modern ; T5 Odia Google ; pa ; modern ; T4.1 Punjabi -Google ; pcm ; modern ; Nigerian Pidgin +Google ; pcm ; moderate ; Nigerian Pidgin Google ; pl ; modern ; T1 Polish Google ; ps ; modern ; T5 Pashto Google ; pt ; modern ; T1 Brazilian Portuguese @@ -283,7 +283,7 @@ Apple ; kn ; modern Apple ; ko ; modern Apple ; lt ; modern Apple ; lv ; modern -Apple ; mi ; modern +Apple ; mi ; moderate Apple ; mk ; modern Apple ; ml ; modern Apple ; mr ; modern @@ -485,7 +485,7 @@ Cldr ; cv ; basic Cldr ; en_AU ; modern Cldr ; es_MX ; modern Cldr ; fr_CA ; modern -Cldr ; mi ; modern +Cldr ; mi ; moderate Cldr ; zh_Hant_HK ; modern #Cldr other (from Google) @@ -516,7 +516,7 @@ Cldr ; su ; basic ; Sundanese (script TBD) Cldr ; ks_Deva ; basic ; Kashmiri (Devanagari) Cldr ; sd_Deva ; basic ; Sindhi (Devanagari script) # Cldr ; cad ; basic ; Caddo -Cldr ; pcm ; modern ; Nigerian Pidgin +Cldr ; pcm ; moderate ; Nigerian Pidgin Cldr ; bgc ; basic ; Haryanvi Cldr ; bho ; basic ; Bhojpuri diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java index 98001180d0f..14afd4f0b4c 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java @@ -840,14 +840,12 @@ public void TestFallbackFormat() { public void Test4897() { ExampleGenerator exampleGenerator = getExampleGenerator("it"); + final CLDRFile cldrFile = exampleGenerator.getCldrFile(); for (String xpath : With.in( - exampleGenerator - .getCldrFile() - .iterator( - "//ldml/dates/timeZoneNames", - exampleGenerator.getCldrFile().getComparator()))) { - String value = exampleGenerator.getCldrFile().getStringValue(xpath); + cldrFile.iterator( + "//ldml/dates/timeZoneNames", cldrFile.getComparator()))) { + String value = cldrFile.getStringValue(xpath); String actual = exampleGenerator.getExampleHtml(xpath, value); if (actual == null) { if (!xpath.contains("singleCountries") && !xpath.contains("gmtZeroFormat")) { diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestStandardCodes.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestStandardCodes.java index 34eea77d547..6cc91e02f20 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestStandardCodes.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestStandardCodes.java @@ -1,8 +1,12 @@ package org.unicode.cldr.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Set; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -30,4 +34,12 @@ void testTargetCoverageLevel(final String locale, final String level) { "Expected getTargetCoverageLevel(%s)=%s but was %s", locale, expectLevel, actualLevel)); } + + @Test + void testTimezoneExclusions() { + SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(); + Set timezones = sdi.getCLDRTimezoneCodes(); + assertTrue(timezones.contains("Europe/Andorra")); + assertFalse(timezones.contains("America/Nipigon")); + } }