Skip to content

Commit

Permalink
[release/5.0-preview8] Wasm ICU (dotnet#39739)
Browse files Browse the repository at this point in the history
* ICU integration and asset loading overhaul (dotnet#37971)

This PR overhauls runtime startup/asset loading and adds support for ICU integration.

The mono-config.js format is reworked and simplified, with new functionality added:

    Individual assets can be loaded from one or more remote sources with configurable fallback behavior
    In addition to the existing support for loading assemblies, you can now pre-load arbitrary files into the native heap or into emscripten's virtual file system. VFS support previously only existed in runtime-test.js but now is available to any consumer of dotnet.js.
    Assets can have a virtual path set so that their application-facing path does not necessarily have to match their path on the server.
    One or more ICU data archives can be added to the assets list and will be automatically loaded and used to enable ICU-based globalization support.
    Many configuration knobs that previously required API calls can now be set declaratively in the configuration file (environment variables, etc.)

WasmAppBuilder is updated to add ICUDataFiles and RemoteSources parameters that can be used to add the associated information to the config file declaratively from a msbuild project.

Various adjustments are made to existing tests and test cases so that they will pass with the addition of ICU integration.

Co-authored-by: EgorBo <egorbo@gmail.com>
Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
Co-authored-by: Larry Ewing <lewing@microsoft.com>

* [wasm] Include data archives and timezone data by default (dotnet#39586)

* Add data archive loading to the generic loading logic

* [mono] Update ICU version, disable some tests for Browser (dotnet#39596)

Co-authored-by: Katelyn Gadd <kg@luminance.org>
Co-authored-by: EgorBo <egorbo@gmail.com>
Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
  • Loading branch information
4 people authored Jul 22, 2020
1 parent 191485c commit 3cab6dd
Show file tree
Hide file tree
Showing 43 changed files with 931 additions and 355 deletions.
4 changes: 2 additions & 2 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
<Uri>https://github.com/dotnet/standard</Uri>
<Sha>cfe95a23647c7de1fe1a349343115bd7720d6949</Sha>
</Dependency>
<Dependency Name="Microsoft.NETCore.Runtime.ICU.Transport" Version="5.0.0-preview.8.20364.1">
<Dependency Name="Microsoft.NETCore.Runtime.ICU.Transport" Version="5.0.0-preview.8.20370.1">
<Uri>https://github.com/dotnet/icu</Uri>
<Sha>bf5a3a643815a8a46693d618d08dbc96f353ca9e</Sha>
<Sha>797c523dd8d75096319f3591958f703b8d74d04b</Sha>
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
Expand Down
2 changes: 1 addition & 1 deletion eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@
<!-- ILLink -->
<MicrosoftNETILLinkTasksVersion>5.0.0-preview.3.20363.5</MicrosoftNETILLinkTasksVersion>
<!-- ICU -->
<MicrosoftNETCoreRuntimeICUTransportVersion>5.0.0-preview.8.20364.1</MicrosoftNETCoreRuntimeICUTransportVersion>
<MicrosoftNETCoreRuntimeICUTransportVersion>5.0.0-preview.8.20370.1</MicrosoftNETCoreRuntimeICUTransportVersion>
<!-- Mono LLVM -->
<runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMSdkVersion>9.0.1-alpha.1.20356.1</runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMSdkVersion>
<runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMToolsVersion>9.0.1-alpha.1.20356.1</runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMToolsVersion>
Expand Down
3 changes: 2 additions & 1 deletion eng/liveBuilds.targets
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,8 @@
Include="
$(LibrariesNativeArtifactsPath)dotnet.js;
$(LibrariesNativeArtifactsPath)dotnet.wasm;
$(LibrariesNativeArtifactsPath)dotnet.timezones.blat;"
$(LibrariesNativeArtifactsPath)dotnet.timezones.blat;
$(LibrariesNativeArtifactsPath)icudt.dat;"
IsNative="true" />
</ItemGroup>

Expand Down
14 changes: 10 additions & 4 deletions eng/testing/tests.mobile.targets
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

<PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
<!-- We need to set this in order to get extensibility on xunit category traits and other arguments we pass down to xunit via MSBuild properties -->
<RunScriptCommand>$HARNESS_RUNNER wasm test --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js -v --output-directory=$XHARNESS_OUT -- --enable-zoneinfo --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
<RunScriptCommand>$HARNESS_RUNNER wasm test --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js -v --output-directory=$XHARNESS_OUT -- --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
</PropertyGroup>

<!-- Generate a self-contained app bundle for Android with tests. -->
Expand Down Expand Up @@ -121,10 +121,17 @@
AssemblyFile="$(WasmAppBuilderTasksAssemblyPath)" />

<Target Condition="'$(TargetOS)' == 'Browser'" Name="BundleTestWasmApp">
<ItemGroup>
<WasmSatelliteAssemblies Include="$(PublishDir)*\*.resources.dll" />
<WasmSatelliteAssemblies>
<CultureName>$([System.IO.Directory]::GetParent('%(Identity)').Name)</CultureName>
</WasmSatelliteAssemblies>
</ItemGroup>
<ItemGroup>
<AssemblySearchPaths Include="$(PublishDir)"/>
<WasmFilesToIncludeInFileSystem Include="@(ContentWithTargetPath)" />
<WasmFilesToIncludeInFileSystem Include="@(ReferenceCopyLocalPaths)" Condition="'%(ReferenceCopyLocalPaths.BuildReference)' == 'true'" />
<WasmFilesToIncludeInFileSystem Include="@(ReferenceCopyLocalPaths)" Condition="'%(ReferenceCopyLocalPaths.BuildReference)' == 'true' and !$([System.String]::new('%(ReferenceCopyLocalPaths.Identity)').EndsWith('.resources.dll'))" />
<WasmFilesToIncludeInFileSystem Include="@(WasmSatelliteAssemblies)" TargetPath="%(WasmSatelliteAssemblies.CultureName)\%(WasmSatelliteAssemblies.Filename)%(WasmSatelliteAssemblies.Extension)" />
<ExtraAssemblies Include="$(PublishDir)$(AssemblyName).dll" />
<!-- we need to preserve these facades for BinaryFormatter tests -->
<ExtraAssemblies Include="$(PublishDir)mscorlib.dll" />
Expand All @@ -140,7 +147,6 @@
<ExtraAssemblies Include="$(PublishDir)System.Xml.dll" />
<ExtraAssemblies Include="$(PublishDir)WindowsBase.dll" />
</ItemGroup>

<Error Condition="!Exists('$(MicrosoftNetCoreAppRuntimePackRidDir)')" Text="MicrosoftNetCoreAppRuntimePackRidDir=$(MicrosoftNetCoreAppRuntimePackRidDir) doesn't exist" />
<WasmAppBuilder
AppDir="$(BundleDir)"
Expand All @@ -149,7 +155,7 @@
MainJS="$(MonoProjectRoot)\wasm\runtime-test.js"
ExtraAssemblies="@(ExtraAssemblies)"
FilesToIncludeInFileSystem="@(WasmFilesToIncludeInFileSystem)"
AssemblySearchPaths="@(AssemblySearchPaths)" />
AssemblySearchPaths="@(AssemblySearchPaths)"/>
</Target>

<Target Name="AddTestRunnersToPublishedFiles"
Expand Down
11 changes: 9 additions & 2 deletions src/libraries/Common/src/Interop/Interop.Calendar.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ internal static partial class Interop
{
internal static partial class Globalization
{
internal delegate void EnumCalendarInfoCallback(
[MarshalAs(UnmanagedType.LPWStr)] string calendarString,
internal unsafe delegate void EnumCalendarInfoCallback(
char* calendarString,
IntPtr context);

[DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_GetCalendars")]
Expand All @@ -19,8 +19,15 @@ internal delegate void EnumCalendarInfoCallback(
[DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_GetCalendarInfo")]
internal static extern unsafe ResultCode GetCalendarInfo(string localeName, CalendarId calendarId, CalendarDataType calendarDataType, char* result, int resultCapacity);

#if TARGET_BROWSER
// Temp workaround for pinvoke callbacks for Mono-Wasm-Interpreter
// https://github.com/dotnet/runtime/issues/39100
[DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_EnumCalendarInfo")]
internal static extern bool EnumCalendarInfo(IntPtr callback, string localeName, CalendarId calendarId, CalendarDataType calendarDataType, IntPtr context);
#else
[DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_EnumCalendarInfo")]
internal static extern bool EnumCalendarInfo(EnumCalendarInfoCallback callback, string localeName, CalendarId calendarId, CalendarDataType calendarDataType, IntPtr context);
#endif

[DllImport(Libraries.GlobalizationNative, EntryPoint = "GlobalizationNative_GetLatestJapaneseEra")]
internal static extern int GetLatestJapaneseEra();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

internal static partial class Interop
{
internal static partial class Globalization
{
// needs to be kept in sync with TimeZoneDisplayNameType in System.Globalization.Native
internal enum TimeZoneDisplayNameType
{
Generic = 0,
Standard = 1,
DaylightSavings = 2,
}
}
}
8 changes: 0 additions & 8 deletions src/libraries/Common/src/Interop/Interop.TimeZoneInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,6 @@ internal static partial class Interop
{
internal static partial class Globalization
{
// needs to be kept in sync with TimeZoneDisplayNameType in System.Globalization.Native
internal enum TimeZoneDisplayNameType
{
Generic = 0,
Standard = 1,
DaylightSavings = 2,
}

[DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_GetTimeZoneDisplayName")]
internal static extern unsafe ResultCode GetTimeZoneDisplayName(
string localeName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@

// All ICU headers need to be included here so that all function prototypes are
// available before the function pointers are declared below.
#include <unicode/uclean.h>
#include <unicode/ucurr.h>
#include <unicode/ucal.h>
#include <unicode/uchar.h>
#include <unicode/ucol.h>
#include <unicode/udat.h>
#include <unicode/udata.h>
#include <unicode/udatpg.h>
#include <unicode/uenum.h>
#include <unicode/uidna.h>
Expand Down Expand Up @@ -55,6 +57,7 @@

#include "pal_compiler.h"

#if !defined(STATIC_ICU)
// List of all functions from the ICU libraries that are used in the System.Globalization.Native.so
#define FOR_ALL_UNCONDITIONAL_ICU_FUNCTIONS \
PER_FUNCTION_BLOCK(u_charsToUChars, libicuuc) \
Expand Down Expand Up @@ -279,3 +282,5 @@ FOR_ALL_ICU_FUNCTIONS
#define usearch_getMatchedLength(...) usearch_getMatchedLength_ptr(__VA_ARGS__)
#define usearch_last(...) usearch_last_ptr(__VA_ARGS__)
#define usearch_openFromCollator(...) usearch_openFromCollator_ptr(__VA_ARGS__)

#endif // !defined(STATIC_ICU)
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
//

#include <stdlib.h>
#include <stdio.h>
#include "pal_icushim_internal.h"
#include "pal_icushim.h"
#include <unicode/putil.h>
#include <unicode/uversion.h>
#include <unicode/localpointer.h>
#include <unicode/utrace.h>

static void log_icu_error(const char* name, UErrorCode status)
{
const char * statusText = u_errorName(status);
fprintf(stderr, "ICU call %s failed with error #%d '%s'.\n", name, status, statusText);
}

static void U_CALLCONV icu_trace_data(const void* context, int32_t fnNumber, int32_t level, const char* fmt, va_list args)
{
char buf[1000];
utrace_vformat(buf, sizeof(buf), 0, fmt, args);
printf("[ICUDT] %s: %s\n", utrace_functionName(fnNumber), buf);
}

#ifdef __EMSCRIPTEN__
#include <emscripten.h>

EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void * pData);

EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void * pData)
{
UErrorCode status = 0;
udata_setCommonData(pData, &status);

if (U_FAILURE(status)) {
log_icu_error("udata_setCommonData", status);
return 0;
} else {
//// Uncomment to enable ICU tracing,
//// see https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/tracing.md
// utrace_setFunctions(0, 0, 0, icu_trace_data);
// utrace_setLevel(UTRACE_VERBOSE);
return 1;
}
}
#endif

int32_t GlobalizationNative_LoadICU(void)
{
const char* icudir = getenv("DOTNET_ICU_DIR");
if (icudir)
u_setDataDirectory(icudir);
else
; // default ICU search path behavior will be used, see http://userguide.icu-project.org/icudata

UErrorCode status = 0;
UVersionInfo version;
// Request the CLDR version to perform basic ICU initialization and find out
// whether it worked.
ulocdata_getCLDRVersion(version, &status);

if (U_FAILURE(status)) {
log_icu_error("ulocdata_getCLDRVersion", status);
return 0;
}

return 1;
}

void GlobalizationNative_InitICUFunctions(void* icuuc, void* icuin, const char* version, const char* suffix)
{
// no-op for static
}

int32_t GlobalizationNative_GetICUVersion(void)
{
UVersionInfo versionInfo;
u_getVersion(versionInfo);

return (versionInfo[0] << 24) + (versionInfo[1] << 16) + (versionInfo[2] << 8) + versionInfo[3];
}
1 change: 1 addition & 0 deletions src/libraries/Native/native-binplace.proj
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
<BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)dotnet.js" />
<BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)dotnet.wasm" />
<BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)dotnet.timezones.blat" />
<BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)icudt.dat" />
<FileWrites Include="@(BinPlaceItem)" />
</ItemGroup>
</Target>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ namespace System.Globalization.Tests
public class TaiwanCalendarDaysAndMonths
{
[Fact]
[ActiveIssue("https://github.com/dotnet/runtime/issues/39285", TestPlatforms.Browser)]
public void DayNames_MonthNames()
{
string[] expectedDayNames =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,18 @@ public class CultureInfoEnglishName
public static IEnumerable<object[]> EnglishName_TestData()
{
yield return new object[] { CultureInfo.CurrentCulture.Name, CultureInfo.CurrentCulture.EnglishName };
yield return new object[] { "en-US", "English (United States)" };
yield return new object[] { "fr-FR", "French (France)" };

if (PlatformDetection.IsNotBrowser)
{
yield return new object[] { "en-US", "English (United States)" };
yield return new object[] { "fr-FR", "French (France)" };
}
else
{
// Browser's ICU doesn't contain CultureInfo.EnglishName
yield return new object[] { "en-US", "en (US)" };
yield return new object[] { "fr-FR", "fr (FR)" };
}
}

[Theory]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,18 @@ public class CultureInfoNativeName
public static IEnumerable<object[]> NativeName_TestData()
{
yield return new object[] { CultureInfo.CurrentCulture.Name, CultureInfo.CurrentCulture.NativeName };
yield return new object[] { "en-US", "English (United States)" };
yield return new object[] { "en-CA", "English (Canada)" };

if (PlatformDetection.IsNotBrowser)
{
yield return new object[] { "en-US", "English (United States)" };
yield return new object[] { "en-CA", "English (Canada)" };
}
else
{
// Browser's ICU doesn't contain CultureInfo.NativeName
yield return new object[] { "en-US", "en (US)" };
yield return new object[] { "en-CA", "en (CA)" };
}
}

[Theory]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,16 @@ public static IEnumerable<object[]> CalendarWeekRule_Get_TestData()
{
yield return new object[] { DateTimeFormatInfo.InvariantInfo, CalendarWeekRule.FirstDay };
yield return new object[] { new CultureInfo("en-US").DateTimeFormat, CalendarWeekRule.FirstDay };
yield return new object[] { new CultureInfo("br-FR").DateTimeFormat, DateTimeFormatInfoData.BrFRCalendarWeekRule() };

if (PlatformDetection.IsNotBrowser)
{
yield return new object[] { new CultureInfo("br-FR").DateTimeFormat, DateTimeFormatInfoData.BrFRCalendarWeekRule() };
}
else
{
// "br-FR" is not presented in Browser's ICU. Let's test ru-RU instead.
yield return new object[] { new CultureInfo("ru-RU").DateTimeFormat, CalendarWeekRule.FirstFourDayWeek };
}
}

[Theory]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,12 @@ public void NativeCalendarName_Get_ReturnsExpected(DateTimeFormatInfo dtfi, Cale
try
{
dtfi.Calendar = calendar;
Assert.Equal(nativeCalendarName, dtfi.NativeCalendarName);

if (PlatformDetection.IsNotBrowser)
{
// Browser's ICU doesn't contain NativeCalendarName,
Assert.Equal(nativeCalendarName, dtfi.NativeCalendarName);
}
}
catch
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public static IEnumerable<object[]> CurrencyGroupSizes_TestData()
yield return new object[] { NumberFormatInfo.InvariantInfo, new int[] { 3 } };
yield return new object[] { CultureInfo.GetCultureInfo("en-US").NumberFormat, new int[] { 3 } };

if (!PlatformDetection.IsUbuntu && !PlatformDetection.IsWindows7 && !PlatformDetection.IsWindows8x && !PlatformDetection.IsFedora)
if (PlatformDetection.IsNotBrowser && !PlatformDetection.IsUbuntu && !PlatformDetection.IsWindows7 && !PlatformDetection.IsWindows8x && !PlatformDetection.IsFedora)
{
yield return new object[] { CultureInfo.GetCultureInfo("ur-IN").NumberFormat, new int[] { 3, 2 } };
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,24 @@ public void CurrencyNegativePattern_Get_ReturnsExpected(NumberFormatInfo format,
Assert.Contains(format.CurrencyNegativePattern, acceptablePatterns);
}

public static IEnumerable<object[]> CurrencyNegativePatternTestLocales()
{
yield return new object[] { "en-US" };
yield return new object[] { "en-CA" };
yield return new object[] { "fa-IR" };
yield return new object[] { "fr-CD" };
yield return new object[] { "fr-CA" };

if (PlatformDetection.IsNotBrowser)
{
// Browser's ICU doesn't contain these locales
yield return new object[] { "as" };
yield return new object[] { "es-BO" };
}
}

[Theory]
[InlineData("en-US")]
[InlineData("en-CA")]
[InlineData("fa-IR")]
[InlineData("fr-CD")]
[InlineData("as")]
[InlineData("es-BO")]
[InlineData("fr-CA")]
[MemberData(nameof(CurrencyNegativePatternTestLocales))]
public void CurrencyNegativePattern_Get_ReturnsExpected_ByLocale(string locale)
{
CultureInfo culture;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ public static IEnumerable<object[]> NumberGroupSizes_TestData()
yield return new object[] { NumberFormatInfo.InvariantInfo, new int[] { 3 } };
yield return new object[] { CultureInfo.GetCultureInfo("en-US").NumberFormat, new int[] { 3 } };

// Culture does not exist on Windows 7
if (!PlatformDetection.IsWindows7)
// Culture does not exist on Windows 7 and in Browser's ICU
if (!PlatformDetection.IsWindows7 && PlatformDetection.IsNotBrowser)
{
yield return new object[] { CultureInfo.GetCultureInfo("ur-IN").NumberFormat, NumberFormatInfoData.UrINNumberGroupSizes() };
}
Expand Down
Loading

0 comments on commit 3cab6dd

Please sign in to comment.