Move scalar loops into IndexOfAnyAsciiSearcher (#91937)

dotnet · Sep 19, 2023 · cc89f38 · cc89f38
1 parent e4df432
commit cc89f38
Show file tree

Hide file tree

Showing 9 changed files with 283 additions and 277 deletions.
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/AnyByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/AnyByteSearchValues.cs
@@ -3,94 +3,56 @@
 
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.Arm;
+using System.Runtime.Intrinsics.Wasm;
+using System.Runtime.Intrinsics.X86;
 
 namespace System.Buffers
 {
     internal sealed class AnyByteSearchValues : SearchValues<byte>
     {
-        private Vector512<byte> _bitmaps;
-        private readonly BitVector256 _lookup;
+        private IndexOfAnyAsciiSearcher.AnyByteState _state;
 
-        public AnyByteSearchValues(ReadOnlySpan<byte> values)
-        {
-            IndexOfAnyAsciiSearcher.ComputeBitmap256(values, out Vector256<byte> bitmap0, out Vector256<byte> bitmap1, out _lookup);
-            _bitmaps = Vector512.Create(bitmap0, bitmap1);
-        }
+        public AnyByteSearchValues(ReadOnlySpan<byte> values) =>
+            IndexOfAnyAsciiSearcher.ComputeAnyByteState(values, out _state);
 
-        internal override byte[] GetValues() => _lookup.GetByteValues();
+        internal override byte[] GetValues() =>
+            _state.Lookup.GetByteValues();
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override bool ContainsCore(byte value) =>
-            _lookup.Contains(value);
+            _state.Lookup.Contains(value);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int IndexOfAny(ReadOnlySpan<byte> span) =>
-            IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.DontNegate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int IndexOfAnyExcept(ReadOnlySpan<byte> span) =>
-            IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.Negate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int LastIndexOfAny(ReadOnlySpan<byte> span) =>
-            LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.DontNegate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int LastIndexOfAnyExcept(ReadOnlySpan<byte> span) =>
-            LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int IndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
-                ? IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmaps)
-                : IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int LastIndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
-                ? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmaps)
-                : LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
-        }
-
-        private int IndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            ref byte searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
-            ref byte cur = ref searchSpace;
-
-            while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
-            {
-                byte b = cur;
-                if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
-                {
-                    return (int)Unsafe.ByteOffset(ref searchSpace, ref cur);
-                }
-
-                cur = ref Unsafe.Add(ref cur, 1);
-            }
-
-            return -1;
-        }
-
-        private int LastIndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            for (int i = searchSpaceLength - 1; i >= 0; i--)
-            {
-                byte b = Unsafe.Add(ref searchSpace, i);
-                if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
-                {
-                    return i;
-                }
-            }
-
-            return -1;
-        }
+            IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.Negate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
     }
 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiByteSearchValues.cs
@@ -3,91 +3,56 @@
 
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.Arm;
+using System.Runtime.Intrinsics.Wasm;
+using System.Runtime.Intrinsics.X86;
 
 namespace System.Buffers
 {
     internal sealed class AsciiByteSearchValues : SearchValues<byte>
     {
-        private Vector256<byte> _bitmap;
-        private readonly BitVector256 _lookup;
+        private IndexOfAnyAsciiSearcher.AsciiState _state;
 
         public AsciiByteSearchValues(ReadOnlySpan<byte> values) =>
-            IndexOfAnyAsciiSearcher.ComputeBitmap(values, out _bitmap, out _lookup);
+            IndexOfAnyAsciiSearcher.ComputeAsciiState(values, out _state);
 
-        internal override byte[] GetValues() => _lookup.GetByteValues();
+        internal override byte[] GetValues() =>
+            _state.Lookup.GetByteValues();
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override bool ContainsCore(byte value) =>
-            _lookup.Contains(value);
+            _state.Lookup.Contains(value);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int IndexOfAny(ReadOnlySpan<byte> span) =>
-            IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int IndexOfAnyExcept(ReadOnlySpan<byte> span) =>
-            IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int LastIndexOfAny(ReadOnlySpan<byte> span) =>
-            LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int LastIndexOfAnyExcept(ReadOnlySpan<byte> span) =>
-            LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int IndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
-                ? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmap)
-                : IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int LastIndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
-                ? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmap)
-                : LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
-        }
-
-        private int IndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            ref byte searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
-            ref byte cur = ref searchSpace;
-
-            while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
-            {
-                byte b = cur;
-                if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
-                {
-                    return (int)Unsafe.ByteOffset(ref searchSpace, ref cur);
-                }
-
-                cur = ref Unsafe.Add(ref cur, 1);
-            }
-
-            return -1;
-        }
-
-        private int LastIndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            for (int i = searchSpaceLength - 1; i >= 0; i--)
-            {
-                byte b = Unsafe.Add(ref searchSpace, i);
-                if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
-                {
-                    return i;
-                }
-            }
-
-            return -1;
-        }
+            IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate>(
+                ref MemoryMarshal.GetReference(span), span.Length, ref _state);
     }
 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiCharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiCharSearchValues.cs
@@ -3,95 +3,57 @@
 
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.Arm;
+using System.Runtime.Intrinsics.Wasm;
+using System.Runtime.Intrinsics.X86;
 
 namespace System.Buffers
 {
     internal sealed class AsciiCharSearchValues<TOptimizations> : SearchValues<char>
         where TOptimizations : struct, IndexOfAnyAsciiSearcher.IOptimizations
     {
-        private Vector256<byte> _bitmap;
-        private readonly BitVector256 _lookup;
+        private IndexOfAnyAsciiSearcher.AsciiState _state;
 
-        public AsciiCharSearchValues(Vector256<byte> bitmap, BitVector256 lookup)
-        {
-            _bitmap = bitmap;
-            _lookup = lookup;
-        }
+        public AsciiCharSearchValues(ReadOnlySpan<char> values) =>
+            IndexOfAnyAsciiSearcher.ComputeAsciiState(values, out _state);
 
-        internal override char[] GetValues() => _lookup.GetCharValues();
+        internal override char[] GetValues() =>
+            _state.Lookup.GetCharValues();
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override bool ContainsCore(char value) =>
-            _lookup.Contains128(value);
+            _state.Lookup.Contains128(value);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int IndexOfAny(ReadOnlySpan<char> span) =>
-            IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate, TOptimizations>(
+                ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
-            IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate, TOptimizations>(
+                ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int LastIndexOfAny(ReadOnlySpan<char> span) =>
-            LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
+            IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate, TOptimizations>(
+                ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
 
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal override int LastIndexOfAnyExcept(ReadOnlySpan<char> span) =>
-            LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int IndexOfAny<TNegator>(ref char searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= Vector128<short>.Count
-                ? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator, TOptimizations>(ref Unsafe.As<char, short>(ref searchSpace), searchSpaceLength, ref _bitmap)
-                : IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int LastIndexOfAny<TNegator>(ref char searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= Vector128<short>.Count
-                ? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref Unsafe.As<char, short>(ref searchSpace), searchSpaceLength, ref _bitmap)
-                : LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
-        }
-
-        private int IndexOfAnyScalar<TNegator>(ref char searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            ref char searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
-            ref char cur = ref searchSpace;
-
-            while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
-            {
-                char c = cur;
-                if (TNegator.NegateIfNeeded(_lookup.Contains128(c)))
-                {
-                    return (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref cur) / sizeof(char));
-                }
-
-                cur = ref Unsafe.Add(ref cur, 1);
-            }
-
-            return -1;
-        }
-
-        private int LastIndexOfAnyScalar<TNegator>(ref char searchSpace, int searchSpaceLength)
-            where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
-        {
-            for (int i = searchSpaceLength - 1; i >= 0; i--)
-            {
-                char c = Unsafe.Add(ref searchSpace, i);
-                if (TNegator.NegateIfNeeded(_lookup.Contains128(c)))
-                {
-                    return i;
-                }
-            }
-
-            return -1;
-        }
+            IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate, TOptimizations>(
+                ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
     }
 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/BitVector256.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/BitVector256.cs
@@ -11,6 +11,18 @@ internal unsafe struct BitVector256
     {
         private fixed uint _values[8];
 
+        public readonly BitVector256 CreateInverse()
+        {
+            BitVector256 inverse = default;
+
+            for (int i = 0; i < 8; i++)
+            {
+                inverse._values[i] = ~_values[i];
+            }
+
+            return inverse;
+        }
+
         public void Set(int c)
         {
             Debug.Assert(c < 256);