Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move SearchValues scalar loops into IndexOfAnyAsciiSearcher #91937

Merged
merged 1 commit into from
Sep 19, 2023
Merged

Move SearchValues scalar loops into IndexOfAnyAsciiSearcher #91937

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 29 additions & 67 deletions src/libraries/System.Private.CoreLib/src/System/SearchValues/AnyByteSearchValues.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,94 +3,56 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;

namespace System.Buffers
{
internal sealed class AnyByteSearchValues : SearchValues<byte>
{
private Vector512<byte> _bitmaps;
private readonly BitVector256 _lookup;
private IndexOfAnyAsciiSearcher.AnyByteState _state;

public AnyByteSearchValues(ReadOnlySpan<byte> values)
{
IndexOfAnyAsciiSearcher.ComputeBitmap256(values, out Vector256<byte> bitmap0, out Vector256<byte> bitmap1, out _lookup);
_bitmaps = Vector512.Create(bitmap0, bitmap1);
}
public AnyByteSearchValues(ReadOnlySpan<byte> values) =>
IndexOfAnyAsciiSearcher.ComputeAnyByteState(values, out _state);

internal override byte[] GetValues() => _lookup.GetByteValues();
internal override byte[] GetValues() =>
_state.Lookup.GetByteValues();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override bool ContainsCore(byte value) =>
_lookup.Contains(value);
_state.Lookup.Contains(value);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int IndexOfAny(ReadOnlySpan<byte> span) =>
IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.DontNegate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int IndexOfAnyExcept(ReadOnlySpan<byte> span) =>
IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.Negate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int LastIndexOfAny(ReadOnlySpan<byte> span) =>
LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.DontNegate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int LastIndexOfAnyExcept(ReadOnlySpan<byte> span) =>
LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int IndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmaps)
: IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int LastIndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmaps)
: LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

private int IndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
ref byte searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
ref byte cur = ref searchSpace;

while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
{
byte b = cur;
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
{
return (int)Unsafe.ByteOffset(ref searchSpace, ref cur);
}

cur = ref Unsafe.Add(ref cur, 1);
}

return -1;
}

private int LastIndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
for (int i = searchSpaceLength - 1; i >= 0; i--)
{
byte b = Unsafe.Add(ref searchSpace, i);
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
{
return i;
}
}

return -1;
}
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.Negate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
}
}
91 changes: 28 additions & 63 deletions src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiByteSearchValues.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,91 +3,56 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;

namespace System.Buffers
{
internal sealed class AsciiByteSearchValues : SearchValues<byte>
{
private Vector256<byte> _bitmap;
private readonly BitVector256 _lookup;
private IndexOfAnyAsciiSearcher.AsciiState _state;

public AsciiByteSearchValues(ReadOnlySpan<byte> values) =>
IndexOfAnyAsciiSearcher.ComputeBitmap(values, out _bitmap, out _lookup);
IndexOfAnyAsciiSearcher.ComputeAsciiState(values, out _state);

internal override byte[] GetValues() => _lookup.GetByteValues();
internal override byte[] GetValues() =>
_state.Lookup.GetByteValues();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override bool ContainsCore(byte value) =>
_lookup.Contains(value);
_state.Lookup.Contains(value);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int IndexOfAny(ReadOnlySpan<byte> span) =>
IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int IndexOfAnyExcept(ReadOnlySpan<byte> span) =>
IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int LastIndexOfAny(ReadOnlySpan<byte> span) =>
LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int LastIndexOfAnyExcept(ReadOnlySpan<byte> span) =>
LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int IndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmap)
: IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int LastIndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmap)
: LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

private int IndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
ref byte searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
ref byte cur = ref searchSpace;

while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
{
byte b = cur;
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
{
return (int)Unsafe.ByteOffset(ref searchSpace, ref cur);
}

cur = ref Unsafe.Add(ref cur, 1);
}

return -1;
}

private int LastIndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
for (int i = searchSpaceLength - 1; i >= 0; i--)
{
byte b = Unsafe.Add(ref searchSpace, i);
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
{
return i;
}
}

return -1;
}
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate>(
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
}
}
96 changes: 29 additions & 67 deletions src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiCharSearchValues.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,95 +3,57 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;

namespace System.Buffers
{
internal sealed class AsciiCharSearchValues<TOptimizations> : SearchValues<char>
where TOptimizations : struct, IndexOfAnyAsciiSearcher.IOptimizations
{
private Vector256<byte> _bitmap;
private readonly BitVector256 _lookup;
private IndexOfAnyAsciiSearcher.AsciiState _state;

public AsciiCharSearchValues(Vector256<byte> bitmap, BitVector256 lookup)
{
_bitmap = bitmap;
_lookup = lookup;
}
public AsciiCharSearchValues(ReadOnlySpan<char> values) =>
IndexOfAnyAsciiSearcher.ComputeAsciiState(values, out _state);

internal override char[] GetValues() => _lookup.GetCharValues();
internal override char[] GetValues() =>
_state.Lookup.GetCharValues();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override bool ContainsCore(char value) =>
_lookup.Contains128(value);
_state.Lookup.Contains128(value);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int IndexOfAny(ReadOnlySpan<char> span) =>
IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate, TOptimizations>(
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate, TOptimizations>(
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int LastIndexOfAny(ReadOnlySpan<char> span) =>
LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate, TOptimizations>(
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);

[CompExactlyDependsOn(typeof(Ssse3))]
[CompExactlyDependsOn(typeof(AdvSimd))]
[CompExactlyDependsOn(typeof(PackedSimd))]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal override int LastIndexOfAnyExcept(ReadOnlySpan<char> span) =>
LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int IndexOfAny<TNegator>(ref char searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= Vector128<short>.Count
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator, TOptimizations>(ref Unsafe.As<char, short>(ref searchSpace), searchSpaceLength, ref _bitmap)
: IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int LastIndexOfAny<TNegator>(ref char searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= Vector128<short>.Count
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref Unsafe.As<char, short>(ref searchSpace), searchSpaceLength, ref _bitmap)
: LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
}

private int IndexOfAnyScalar<TNegator>(ref char searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
ref char searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
ref char cur = ref searchSpace;

while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
{
char c = cur;
if (TNegator.NegateIfNeeded(_lookup.Contains128(c)))
{
return (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref cur) / sizeof(char));
}

cur = ref Unsafe.Add(ref cur, 1);
}

return -1;
}

private int LastIndexOfAnyScalar<TNegator>(ref char searchSpace, int searchSpaceLength)
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
{
for (int i = searchSpaceLength - 1; i >= 0; i--)
{
char c = Unsafe.Add(ref searchSpace, i);
if (TNegator.NegateIfNeeded(_lookup.Contains128(c)))
{
return i;
}
}

return -1;
}
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate, TOptimizations>(
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
}
}
12 changes: 12 additions & 0 deletions src/libraries/System.Private.CoreLib/src/System/SearchValues/BitVector256.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,18 @@ internal unsafe struct BitVector256
{
private fixed uint _values[8];

public readonly BitVector256 CreateInverse()
{
BitVector256 inverse = default;

for (int i = 0; i < 8; i++)
{
inverse._values[i] = ~_values[i];
}

return inverse;
}

public void Set(int c)
{
Debug.Assert(c < 256);
Expand Down
Loading