Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify IndexOfAnyAsciiByteValues for needles with 0 on X86 #82866

Merged
merged 1 commit into from
Mar 2, 2023
Merged

Simplify IndexOfAnyAsciiByteValues for needles with 0 on X86 #82866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/libraries/System.Memory/tests/Span/IndexOfAnyValues.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,8 +496,8 @@ private static ReadOnlySpan<T> GetRandomSlice<T>(Random rng, ReadOnlySpan<T> spa
private static void AssertionFailed<T>(ReadOnlySpan<T> haystack, ReadOnlySpan<T> needle, int expected, int actual, string approach)
where T : INumber<T>
{
string readableHaystack = string.Join(", ", haystack.ToString().Select(c => int.CreateChecked(c)));
string readableNeedle = string.Join(", ", needle.ToString().Select(c => int.CreateChecked(c)));
string readableHaystack = string.Join(", ", haystack.ToArray().Select(c => int.CreateChecked(c)));
string readableNeedle = string.Join(", ", needle.ToArray().Select(c => int.CreateChecked(c)));

Assert.True(false, $"Expected {expected}, got {approach}={actual} for needle='{readableNeedle}', haystack='{readableHaystack}'");
}
Expand Down
14 changes: 5 additions & 9 deletions ...libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyAsciiByteValues.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,13 @@

namespace System.Buffers
{
internal sealed class IndexOfAnyAsciiByteValues<TOptimizations> : IndexOfAnyValues<byte>
where TOptimizations : struct, IndexOfAnyAsciiSearcher.IOptimizations
internal sealed class IndexOfAnyAsciiByteValues : IndexOfAnyValues<byte>
{
private readonly Vector128<byte> _bitmap;
private readonly BitVector256 _lookup;

public IndexOfAnyAsciiByteValues(Vector128<byte> bitmap, BitVector256 lookup)
{
_bitmap = bitmap;
_lookup = lookup;
}
public IndexOfAnyAsciiByteValues(ReadOnlySpan<byte> values) =>
IndexOfAnyAsciiSearcher.ComputeBitmap(values, out _bitmap, out _lookup);

internal override byte[] GetValues() => _lookup.GetByteValues();

Expand Down Expand Up @@ -46,7 +42,7 @@ private int IndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator, TOptimizations>(ref searchSpace, searchSpaceLength, _bitmap)
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, _bitmap)
: IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

Expand All @@ -55,7 +51,7 @@ private int LastIndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref searchSpace, searchSpaceLength, _bitmap)
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, _bitmap)
: LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

Expand Down
56 changes: 10 additions & 46 deletions src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyAsciiSearcher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,9 +384,8 @@ internal static int LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref short
return -1;
}

internal static int IndexOfAnyVectorized<TNegator, TOptimizations>(ref byte searchSpace, int searchSpaceLength, Vector128<byte> bitmap)
internal static int IndexOfAnyVectorized<TNegator>(ref byte searchSpace, int searchSpaceLength, Vector128<byte> bitmap)
where TNegator : struct, INegator
where TOptimizations : struct, IOptimizations
{
ref byte currentSearchSpace = ref searchSpace;

Expand All @@ -408,7 +407,7 @@ internal static int IndexOfAnyVectorized<TNegator, TOptimizations>(ref byte sear
{
Vector256<byte> source = Vector256.LoadUnsafe(ref currentSearchSpace);

Vector256<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap256);
Vector256<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap256));
if (result != Vector256<byte>.Zero)
{
return ComputeFirstIndex<byte, TNegator>(ref searchSpace, ref currentSearchSpace, result);
Expand All @@ -433,7 +432,7 @@ internal static int IndexOfAnyVectorized<TNegator, TOptimizations>(ref byte sear
Vector128<byte> source1 = Vector128.LoadUnsafe(ref halfVectorAwayFromEnd);
Vector256<byte> source = Vector256.Create(source0, source1);

Vector256<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap256);
Vector256<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap256));
if (result != Vector256<byte>.Zero)
{
return ComputeFirstIndexOverlapped<byte, TNegator>(ref searchSpace, ref firstVector, ref halfVectorAwayFromEnd, result);
Expand All @@ -454,7 +453,7 @@ internal static int IndexOfAnyVectorized<TNegator, TOptimizations>(ref byte sear
{
Vector128<byte> source = Vector128.LoadUnsafe(ref currentSearchSpace);

Vector128<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap);
Vector128<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap));
if (result != Vector128<byte>.Zero)
{
return ComputeFirstIndex<byte, TNegator>(ref searchSpace, ref currentSearchSpace, result);
Expand All @@ -480,7 +479,7 @@ internal static int IndexOfAnyVectorized<TNegator, TOptimizations>(ref byte sear
ulong source1 = Unsafe.ReadUnaligned<ulong>(ref halfVectorAwayFromEnd);
Vector128<byte> source = Vector128.Create(source0, source1).AsByte();

Vector128<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap);
Vector128<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap));
if (result != Vector128<byte>.Zero)
{
return ComputeFirstIndexOverlapped<byte, TNegator>(ref searchSpace, ref firstVector, ref halfVectorAwayFromEnd, result);
Expand All @@ -490,9 +489,8 @@ internal static int IndexOfAnyVectorized<TNegator, TOptimizations>(ref byte sear
return -1;
}

internal static int LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref byte searchSpace, int searchSpaceLength, Vector128<byte> bitmap)
internal static int LastIndexOfAnyVectorized<TNegator>(ref byte searchSpace, int searchSpaceLength, Vector128<byte> bitmap)
where TNegator : struct, INegator
where TOptimizations : struct, IOptimizations
{
ref byte currentSearchSpace = ref Unsafe.Add(ref searchSpace, searchSpaceLength);

Expand All @@ -516,7 +514,7 @@ internal static int LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref byte

Vector256<byte> source = Vector256.LoadUnsafe(ref currentSearchSpace);

Vector256<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap256);
Vector256<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap256));
if (result != Vector256<byte>.Zero)
{
return ComputeLastIndex<byte, TNegator>(ref searchSpace, ref currentSearchSpace, result);
Expand All @@ -539,7 +537,7 @@ internal static int LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref byte
Vector128<byte> source1 = Vector128.LoadUnsafe(ref secondVector);
Vector256<byte> source = Vector256.Create(source0, source1);

Vector256<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap256);
Vector256<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap256));
if (result != Vector256<byte>.Zero)
{
return ComputeLastIndexOverlapped<byte, TNegator>(ref searchSpace, ref secondVector, result);
Expand All @@ -562,7 +560,7 @@ internal static int LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref byte

Vector128<byte> source = Vector128.LoadUnsafe(ref currentSearchSpace);

Vector128<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap);
Vector128<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap));
if (result != Vector128<byte>.Zero)
{
return ComputeLastIndex<byte, TNegator>(ref searchSpace, ref currentSearchSpace, result);
Expand All @@ -586,7 +584,7 @@ internal static int LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref byte
ulong source1 = Unsafe.ReadUnaligned<ulong>(ref secondVector);
Vector128<byte> source = Vector128.Create(source0, source1).AsByte();

Vector128<byte> result = IndexOfAnyLookup<TNegator, TOptimizations>(source, bitmap);
Vector128<byte> result = TNegator.NegateIfNeeded(IndexOfAnyLookupCore(source, bitmap));
if (result != Vector128<byte>.Zero)
{
return ComputeLastIndexOverlapped<byte, TNegator>(ref searchSpace, ref secondVector, result);
Expand Down Expand Up @@ -840,23 +838,6 @@ private static Vector128<byte> IndexOfAnyLookup<TNegator, TOptimizations>(Vector
return TNegator.NegateIfNeeded(result);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> IndexOfAnyLookup<TNegator, TOptimizations>(Vector128<byte> source, Vector128<byte> bitmapLookup)
where TNegator : struct, INegator
where TOptimizations : struct, IOptimizations
{
Vector128<byte> result = IndexOfAnyLookupCore(source, bitmapLookup);

// On X86, values above 127 will map to 0. If 0 is present in the needle, we must clear the false positives.
if (TOptimizations.NeedleContainsZero)
{
Vector128<byte> ascii = Vector128.LessThan(source, Vector128.Create((byte)128));
result &= ascii;
}

return TNegator.NegateIfNeeded(result);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> IndexOfAnyLookupCore(Vector128<byte> source, Vector128<byte> bitmapLookup)
{
Expand Down Expand Up @@ -903,23 +884,6 @@ private static Vector256<byte> IndexOfAnyLookup<TNegator, TOptimizations>(Vector
return TNegator.NegateIfNeeded(result);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> IndexOfAnyLookup<TNegator, TOptimizations>(Vector256<byte> source, Vector256<byte> bitmapLookup)
where TNegator : struct, INegator
where TOptimizations : struct, IOptimizations
{
// See comments in IndexOfAnyLookup(Vector128<byte>) above for more details.
Vector256<byte> result = IndexOfAnyLookupCore(source, bitmapLookup);

if (TOptimizations.NeedleContainsZero)
{
Vector256<byte> ascii = Vector256.LessThan(source, Vector256.Create((byte)128));
result &= ascii;
}

return TNegator.NegateIfNeeded(result);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> IndexOfAnyLookupCore(Vector256<byte> source, Vector256<byte> bitmapLookup)
{
Expand Down
6 changes: 1 addition & 5 deletions src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,7 @@ public static IndexOfAnyValues<byte> Create(ReadOnlySpan<byte> values)

if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && maxInclusive < 128)
{
IndexOfAnyAsciiSearcher.ComputeBitmap(values, out Vector128<byte> bitmap, out BitVector256 lookup);

return Ssse3.IsSupported && lookup.Contains(0)
? new IndexOfAnyAsciiByteValues<IndexOfAnyAsciiSearcher.Ssse3HandleZeroInNeedle>(bitmap, lookup)
: new IndexOfAnyAsciiByteValues<IndexOfAnyAsciiSearcher.Default>(bitmap, lookup);
return new IndexOfAnyAsciiByteValues(values);
}

return new IndexOfAnyByteValues(values);
Expand Down