Big endian fixes for dotnet runtime

src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs

@@ -200,6 +200,10 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Default(byte* pBuffer, n
                 currentUInt32 = Unsafe.ReadUnaligned<ushort>(pBuffer);
                 if (!AllBytesInUInt32AreAscii(currentUInt32))
                 {
+                    if (!BitConverter.IsLittleEndian)
+                    {
+                        currentUInt32 = currentUInt32 << 16;
+                    }
                     goto FoundNonAsciiData;
                 }
 
@@ -1678,6 +1682,10 @@ public static unsafe nuint WidenAsciiToUtf16(byte* pAsciiBuffer, char* pUtf16Buf
                 asciiData = Unsafe.ReadUnaligned<ushort>(pAsciiBuffer + currentOffset);
                 if (!AllBytesInUInt32AreAscii(asciiData))
                 {
+                    if (!BitConverter.IsLittleEndian)
+                    {
+                        asciiData = asciiData << 16;
+                    }
                     goto FoundNonAsciiData;
                 }
 
@@ -1719,11 +1727,23 @@ public static unsafe nuint WidenAsciiToUtf16(byte* pAsciiBuffer, char* pUtf16Buf
 
             // Drain ASCII bytes one at a time.
 
-            while (((byte)asciiData & 0x80) == 0)
+            if (BitConverter.IsLittleEndian)
             {
-                pUtf16Buffer[currentOffset] = (char)(byte)asciiData;
-                currentOffset++;
-                asciiData >>= 8;
+                while (((byte)asciiData & 0x80) == 0)
+                {
+                    pUtf16Buffer[currentOffset] = (char)(byte)asciiData;
+                    currentOffset++;
+                    asciiData >>= 8;
+                }
+            }
+            else
+            {
+                while ((asciiData & 0x80000000) == 0)
+                {
+                    asciiData = BitOperations.RotateLeft(asciiData, 8);
+                    pUtf16Buffer[currentOffset] = (char)(byte)asciiData;
+                    currentOffset++;
+                }
             }
 
             goto Finish;

src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Helpers.cs

@@ -143,7 +143,7 @@ private static uint ExtractFourUtf8BytesFromSurrogatePair(uint value)
                 tempB |= tempA;
 
                 uint tempC = (value << 2) & 0x0000_0F00u; // = [ 00000000 00000000 0000yyyy 00000000 ]
-                uint tempD = (value >> 6) & 0x0003_0000u; // = [ 00000000 00000000 00yy0000 00000000 ]
+                uint tempD = (value >> 4) & 0x0000_3000u; // = [ 00000000 00000000 00yy0000 00000000 ]
                 tempD |= tempC;
 
                 uint tempE = (value & 0x3Fu) + 0xF080_8080u; // = [ 11110000 10000000 10000000 10xxxxxx ]
@@ -232,7 +232,7 @@ private static uint ExtractUtf8TwoByteSequenceFromFirstUtf16Char(uint value)
                 // want to return [ ######## ######## 110yyyyy 10xxxxxx ]
 
                 uint temp = (value >> 16) & 0x3Fu; // [ 00000000 00000000 00000000 00xxxxxx ]
-                value = (value >> 22) & 0x1F00u; // [ 00000000 00000000 000yyyyy 0000000 ]
+                value = (value >> 14) & 0x1F00u; // [ 00000000 00000000 000yyyyy 0000000 ]
                 return value + temp + 0xC080u;
             }
         }
@@ -498,7 +498,7 @@ private static bool UInt32BeginsWithUtf8FourByteMask(uint value)
             // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207.
 
             return (BitConverter.IsLittleEndian && (((value - 0x8080_80F0u) & 0xC0C0_C0F8u) == 0))
-                || (!BitConverter.IsLittleEndian && (((value - 0xF080_8000u) & 0xF8C0_C0C0u) == 0));
+                || (!BitConverter.IsLittleEndian && (((value - 0xF080_8080u) & 0xF8C0_C0C0u) == 0));
         }
 
         /// <summary>

src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs

@@ -1134,7 +1134,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt
                     }
                     else
                     {
-                        pOutputBuffer[0] = (byte)(thisDWord >> 24); // extract [ AA 00 ## ## ]
+                        pOutputBuffer[0] = (byte)(thisDWord >> 16); // extract [ 00 AA ## ## ]
                     }
 
                     pInputBuffer++;

src/libraries/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/Formatters/Binary/BinaryFormatterWriter.cs

@@ -268,7 +268,6 @@ private void WriteArrayAsBytes(Array array, int typeLength)
                 if (!BitConverter.IsLittleEndian)
                 {
                     // we know that we are writing a primitive type, so just do a simple swap
-                    Debug.Fail("Re-review this code if/when we start running on big endian systems");
                     for (int i = 0; i < bufferUsed; i += typeLength)
                     {
                         for (int j = 0; j < typeLength / 2; j++)

src/libraries/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/Formatters/Binary/BinaryParser.cs

@@ -894,7 +894,6 @@ private void ReadArrayAsBytes(ParseRecord pr)
                     if (!BitConverter.IsLittleEndian)
                     {
                         // we know that we are reading a primitive type, so just do a simple swap
-                        Debug.Fail("Re-review this code if/when we start running on big endian systems");
                         for (int i = 0; i < bufferUsed; i += typeLength)
                         {
                             for (int j = 0; j < typeLength / 2; j++)

src/libraries/System.Text.Encoding.CodePages/src/System.Text.Encoding.CodePages.csproj

@@ -77,10 +77,14 @@
     <Reference Include="System.Collections" />
     <Reference Include="System.Diagnostics.Debug" />
     <Reference Include="System.Diagnostics.Tools" />
+    <Reference Include="System.Memory" />
     <Reference Include="System.Resources.ResourceManager" />
     <Reference Include="System.Runtime" />
     <Reference Include="System.Runtime.Extensions" />
     <Reference Include="System.Runtime.InteropServices" />
     <Reference Include="System.Threading" />
   </ItemGroup>
+  <ItemGroup Condition="!$(TargetFramework.StartsWith('$(NetCoreAppCurrent)'))">
+    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
+  </ItemGroup>
 </Project>

src/libraries/System.Text.Encoding.CodePages/src/System/Text/BaseCodePageEncoding.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Buffers.Binary;
 using System.Reflection;
 using System.IO;
 using System.Diagnostics;
@@ -99,6 +100,28 @@ internal struct CodePageDataFileHeader
             internal short unused1;             // Add an unused WORD so that CodePages is aligned with DWORD boundary.
         }
         private const int CODEPAGE_DATA_FILE_HEADER_SIZE = 44;
+        internal static unsafe void ReadCodePageDataFileHeader(Stream stream, byte[] codePageDataFileHeader)
+        {
+            stream.Read(codePageDataFileHeader, 0, codePageDataFileHeader.Length);
+            if (!BitConverter.IsLittleEndian)
+            {
+                fixed (byte* pBytes = &codePageDataFileHeader[0])
+                {
+                    CodePageDataFileHeader* p = (CodePageDataFileHeader*)pBytes;
+                    char *pTableName = &p->TableName;
+                    for (int i = 0; i < 16; i++)
+                    {
+                            pTableName[i] = (char)BinaryPrimitives.ReverseEndianness((ushort)pTableName[i]);
+                    }
+                    ushort *pVersion = &p->Version;
+                    for (int i = 0; i < 4; i++)
+                    {
+                            pVersion[i] = BinaryPrimitives.ReverseEndianness(pVersion[i]);
+                    }
+                    p->CodePageCount = BinaryPrimitives.ReverseEndianness(p->CodePageCount);
+                }
+            }
+        }
 
         [StructLayout(LayoutKind.Explicit, Pack = 2)]
         internal unsafe struct CodePageIndex
@@ -112,6 +135,25 @@ internal unsafe struct CodePageIndex
             [FieldOffset(0x24)]
             internal int Offset;            // DWORD
         }
+        internal static unsafe void ReadCodePageIndex(Stream stream, byte[] codePageIndex)
+        {
+            stream.Read(codePageIndex, 0, codePageIndex.Length);
+            if (!BitConverter.IsLittleEndian)
+            {
+                fixed (byte* pBytes = &codePageIndex[0])
+                {
+                    CodePageIndex* p = (CodePageIndex*)pBytes;
+                    char *pCodePageName = &p->CodePageName;
+                    for (int i = 0; i < 16; i++)
+                    {
+                            pCodePageName[i] = (char)BinaryPrimitives.ReverseEndianness((ushort)pCodePageName[i]);
+                    }
+                    p->CodePage = BinaryPrimitives.ReverseEndianness(p->CodePage);
+                    p->ByteCount = BinaryPrimitives.ReverseEndianness(p->ByteCount);
+                    p->Offset = BinaryPrimitives.ReverseEndianness(p->Offset);
+                }
+            }
+        }
 
         [StructLayout(LayoutKind.Explicit)]
         internal unsafe struct CodePageHeader
@@ -136,6 +178,30 @@ internal unsafe struct CodePageHeader
             internal ushort ByteReplace;    // WORD     // default replacement bytes
         }
         private const int CODEPAGE_HEADER_SIZE = 48;
+        internal static unsafe void ReadCodePageHeader(Stream stream, byte[] codePageHeader)
+        {
+            stream.Read(codePageHeader, 0, codePageHeader!.Length);
+            if (!BitConverter.IsLittleEndian)
+            {
+                fixed (byte* pBytes = &codePageHeader[0])
+                {
+                    CodePageHeader* p = (CodePageHeader*)pBytes;
+                    char *pCodePageName = &p->CodePageName;
+                    for (int i = 0; i < 16; i++)
+                    {
+                            pCodePageName[i] = (char)BinaryPrimitives.ReverseEndianness((ushort)pCodePageName[i]);
+                    }
+                    p->VersionMajor = BinaryPrimitives.ReverseEndianness(p->VersionMajor);
+                    p->VersionMinor = BinaryPrimitives.ReverseEndianness(p->VersionMinor);
+                    p->VersionRevision = BinaryPrimitives.ReverseEndianness(p->VersionRevision);
+                    p->VersionBuild = BinaryPrimitives.ReverseEndianness(p->VersionBuild);
+                    p->CodePage = BinaryPrimitives.ReverseEndianness(p->CodePage);
+                    p->ByteCount = BinaryPrimitives.ReverseEndianness(p->ByteCount);
+                    p->UnicodeReplace = (char)BinaryPrimitives.ReverseEndianness((ushort)p->UnicodeReplace);
+                    p->ByteReplace = BinaryPrimitives.ReverseEndianness(p->ByteReplace);
+                }
+            }
+        }
 
         // Initialize our global stuff
         private static readonly byte[] s_codePagesDataHeader = new byte[CODEPAGE_DATA_FILE_HEADER_SIZE];
@@ -166,7 +232,7 @@ internal static Stream GetEncodingDataStream(string tableName)
             }
 
             // Read the header
-            stream.Read(s_codePagesDataHeader, 0, s_codePagesDataHeader.Length);
+            ReadCodePageDataFileHeader(stream, s_codePagesDataHeader);
 
             return stream;
         }
@@ -210,14 +276,14 @@ private unsafe bool FindCodePage(int codePage)
                     CodePageIndex* pCodePageIndex = (CodePageIndex*)pBytes;
                     for (int i = 0; i < codePagesCount; i++)
                     {
-                        s_codePagesEncodingDataStream.Read(codePageIndex, 0, codePageIndex.Length);
+                        ReadCodePageIndex(s_codePagesEncodingDataStream, codePageIndex);
 
                         if (pCodePageIndex->CodePage == codePage)
                         {
                             // Found it!
                             long position = s_codePagesEncodingDataStream.Position;
                             s_codePagesEncodingDataStream.Seek((long)pCodePageIndex->Offset, SeekOrigin.Begin);
-                            s_codePagesEncodingDataStream.Read(m_codePageHeader, 0, m_codePageHeader!.Length);
+                            ReadCodePageHeader(s_codePagesEncodingDataStream, m_codePageHeader);
                             m_firstDataWordOffset = (int)s_codePagesEncodingDataStream.Position; // stream now pointing to the codepage data
 
                             if (i == codePagesCount - 1) // last codepage
@@ -229,7 +295,7 @@ private unsafe bool FindCodePage(int codePage)
                                 // Read Next codepage data to get the offset and then calculate the size
                                 s_codePagesEncodingDataStream.Seek(position, SeekOrigin.Begin);
                                 int currentOffset = pCodePageIndex->Offset;
-                                s_codePagesEncodingDataStream.Read(codePageIndex, 0, codePageIndex.Length);
+                                ReadCodePageIndex(s_codePagesEncodingDataStream, codePageIndex);
                                 m_dataSize = pCodePageIndex->Offset - currentOffset - m_codePageHeader.Length;
                             }
 
@@ -266,7 +332,7 @@ internal static unsafe int GetCodePageByteSize(int codePage)
                     CodePageIndex* pCodePageIndex = (CodePageIndex*)pBytes;
                     for (int i = 0; i < codePagesCount; i++)
                     {
-                        s_codePagesEncodingDataStream.Read(codePageIndex, 0, codePageIndex.Length);
+                        ReadCodePageIndex(s_codePagesEncodingDataStream, codePageIndex);
 
                         if (pCodePageIndex->CodePage == codePage)
                         {

src/libraries/System.Text.Encoding.CodePages/src/System/Text/BaseCodePageEncoding.netcoreapp.cs

@@ -2,13 +2,34 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.IO;
+using System.Buffers.Binary;
 using System.Runtime.Serialization;
 using System.Runtime.CompilerServices;
 
 namespace System.Text
 {
     internal abstract partial class BaseCodePageEncoding : EncodingNLS, ISerializable
     {
+        internal static unsafe void ReadCodePageIndex(Stream stream, Span<byte> codePageIndex)
+        {
+            stream.Read(codePageIndex);
+            if (!BitConverter.IsLittleEndian)
+            {
+                fixed (byte* pBytes = &codePageIndex[0])
+                {
+                    CodePageIndex* p = (CodePageIndex*)pBytes;
+                    char *pCodePageName = &p->CodePageName;
+                    for (int i = 0; i < 16; i++)
+                    {
+                            pCodePageName[i] = (char)BinaryPrimitives.ReverseEndianness((ushort)pCodePageName[i]);
+                    }
+                    p->CodePage = BinaryPrimitives.ReverseEndianness(p->CodePage);
+                    p->ByteCount = BinaryPrimitives.ReverseEndianness(p->ByteCount);
+                    p->Offset = BinaryPrimitives.ReverseEndianness(p->Offset);
+                }
+            }
+        }
+
         internal static unsafe EncodingInfo [] GetEncodings(CodePagesEncodingProvider provider)
         {
             lock (s_streamLock)
@@ -29,7 +50,7 @@ internal static unsafe EncodingInfo [] GetEncodings(CodePagesEncodingProvider pr
 
                 for (int i = 0; i < codePagesCount; i++)
                 {
-                    s_codePagesEncodingDataStream.Read(pCodePageIndex);
+                    ReadCodePageIndex(s_codePagesEncodingDataStream, pCodePageIndex);
 
                     string codePageName;
                     switch (codePageIndex.CodePage)