Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ARM64-SVE: Add AddSequentialAcross #104640

Merged
merged 6 commits into from
Jul 13, 2024
Merged

ARM64-SVE: Add AddSequentialAcross #104640

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9b24a75
Add AddSequentialAcross
amanasifkhalid Jul 9, 2024
ae18317
Skip inactive lanes
amanasifkhalid Jul 10, 2024
d1ad2ca
Generate mask data
amanasifkhalid Jul 11, 2024
c6bb59f
Tweak test validation
amanasifkhalid Jul 12, 2024
c3430c7
Remove CndSel scenarios
amanasifkhalid Jul 12, 2024
063c142
Merge from main
amanasifkhalid Jul 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/coreclr/jit/hwintrinsiclistarm64sve.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ HARDWARE_INTRINSIC(Sve, AbsoluteDifference,
HARDWARE_INTRINSIC(Sve, Add, -1, -1, false, {INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, AddAcross, -1, 1, true, {INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_faddv, INS_sve_faddv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, AddSaturate, -1, 2, true, {INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, AddSequentialAcross, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fadda, INS_sve_fadda}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve, And, -1, -1, false, {INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, AndAcross, -1, -1, false, {INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, BitwiseClear, -1, -1, false, {INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down
14 changes: 14 additions & 0 deletions ...ries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,20 @@ internal Arm64() { }
/// </summary>
public static unsafe Vector<ulong> AddSaturate(Vector<ulong> left, Vector<ulong> right) { throw new PlatformNotSupportedException(); }

/// AddSequentialAcross : Add reduction (strictly-ordered)

/// <summary>
/// float64_t svadda[_f64](svbool_t pg, float64_t initial, svfloat64_t op)
/// FADDA Dtied, Pg, Dtied, Zop.D
/// </summary>
public static unsafe Vector<double> AddSequentialAcross(Vector<double> initial, Vector<double> value) { throw new PlatformNotSupportedException(); }

/// <summary>
/// float32_t svadda[_f32](svbool_t pg, float32_t initial, svfloat32_t op)
/// FADDA Stied, Pg, Stied, Zop.S
/// </summary>
public static unsafe Vector<float> AddSequentialAcross(Vector<float> initial, Vector<float> value) { throw new PlatformNotSupportedException(); }

/// And : Bitwise AND

/// <summary>
Expand Down
14 changes: 14 additions & 0 deletions src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,20 @@ internal Arm64() { }
/// </summary>
public static unsafe Vector<ulong> AddSaturate(Vector<ulong> left, Vector<ulong> right) => AddSaturate(left, right);

/// AddSequentialAcross : Add reduction (strictly-ordered)

/// <summary>
/// float64_t svadda[_f64](svbool_t pg, float64_t initial, svfloat64_t op)
/// FADDA Dtied, Pg, Dtied, Zop.D
/// </summary>
public static unsafe Vector<double> AddSequentialAcross(Vector<double> initial, Vector<double> value) => AddSequentialAcross(initial, value);

/// <summary>
/// float32_t svadda[_f32](svbool_t pg, float32_t initial, svfloat32_t op)
/// FADDA Stied, Pg, Stied, Zop.S
/// </summary>
public static unsafe Vector<float> AddSequentialAcross(Vector<float> initial, Vector<float> value) => AddSequentialAcross(initial, value);

/// And : Bitwise AND

/// <summary>
Expand Down
3 changes: 3 additions & 0 deletions src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4253,6 +4253,9 @@ internal Arm64() { }
public static System.Numerics.Vector<uint> AddSaturate(System.Numerics.Vector<uint> left, System.Numerics.Vector<uint> right) { throw null; }
public static System.Numerics.Vector<ulong> AddSaturate(System.Numerics.Vector<ulong> left, System.Numerics.Vector<ulong> right) { throw null; }

public static System.Numerics.Vector<double> AddSequentialAcross(System.Numerics.Vector<double> initial, System.Numerics.Vector<double> value) { throw null; }
public static System.Numerics.Vector<float> AddSequentialAcross(System.Numerics.Vector<float> initial, System.Numerics.Vector<float> value) { throw null; }

public static System.Numerics.Vector<byte> And(System.Numerics.Vector<byte> left, System.Numerics.Vector<byte> right) { throw null; }
public static System.Numerics.Vector<short> And(System.Numerics.Vector<short> left, System.Numerics.Vector<short> right) { throw null; }
public static System.Numerics.Vector<int> And(System.Numerics.Vector<int> left, System.Numerics.Vector<int> right) { throw null; }
Expand Down
3 changes: 3 additions & 0 deletions src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3121,6 +3121,9 @@
("SveVecBinOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve_AddSaturate_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AddSaturate", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "Helpers.AddSaturate(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.AddSaturate(left[i], right[i])"}),
("SveVecBinOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve_AddSaturate_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AddSaturate", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "Helpers.AddSaturate(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.AddSaturate(left[i], right[i])"}),

("SveVecBinOpVecTest.template", new Dictionary<string, string> { ["TestName"] = "Sve_AddSequentialAcross_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AddSequentialAcross", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.AddSequentialAcross(left, right))", ["GetVectorResult"] = "Helpers.AddSequentialAcross(left, right)"}),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this test the operation with ConditionalSelect as well?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does, using SimpleVecOpTest_VectorValidationLogicForCndSel and SimpleVecOpTest_VectorValidationLogicForCndSel_FalseValue. I guess I've been getting lucky with mask[0] being 0 for the CndSel scenarios that skip lanes in the second op.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should use Helpers.getMask*() to produce the mask input.

Copy link
Member Author

@amanasifkhalid amanasifkhalid Jul 11, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, I tweaked the test definitions to use getMask* for NextValueOp1. The only odd thing about this approach is the template for SveVecBinOpVecTest uses NextValueOp1 for generating the mask input, and for op1's input, so the test values for op1 are artificially constrained by this approach. I can tweak the test template to use separate template variables for these two, but lots of existing tests would need to be updated, and I don't think the value gained from doing this is all that much.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am fine with not doing it in this PR, but can you please open an issue to update the templates to have them separate?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure thing: #104804

("SveVecBinOpVecTest.template", new Dictionary<string, string> { ["TestName"] = "Sve_AddSequentialAcross_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AddSequentialAcross", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.AddSequentialAcross(left, right))", ["GetVectorResult"] = "Helpers.AddSequentialAcross(left, right)"}),

("SveVecBinOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve_And_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "And", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(sbyte)TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "Helpers.And(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.And(left[i], right[i])"}),
("SveVecBinOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve_And_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "And", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(short)TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "Helpers.And(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.And(left[i], right[i])"}),
("SveVecBinOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve_And_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "And", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "Helpers.And(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.And(left[i], right[i])"}),
Expand Down
26 changes: 26 additions & 0 deletions src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4136,6 +4136,32 @@ public static byte AddSaturate(byte op1, byte op2)
return ovf ? byte.MaxValue : result;
}

public static double[] AddSequentialAcross(double[] op1, double[] op2)
{
double[] result = new double[op1.Length];
result[0] = op1[0];

foreach (double num in op2)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This only calculates the sum for active lanes, so you need to check that as well.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, fixed.

{
result[0] += num;
}

return result;
}

public static float[] AddSequentialAcross(float[] op1, float[] op2)
{
float[] result = new float[op1.Length];
result[0] = op1[0];

foreach (float num in op2)
{
result[0] += num;
}

return result;
}

public static sbyte NegateSaturate(sbyte op1) => SubtractSaturate((sbyte)0, op1);

public static sbyte SubtractSaturate(sbyte op1, sbyte op2)
Expand Down
Loading