Skip to content

Commit

Permalink
[Arm64] ASIMD Shift Intrinsics (#36830)
Browse files Browse the repository at this point in the history 
* ShiftArithmetic

* ShiftArithmeticRounded

* ShiftArithmeticRoundedSaturate

* ShiftArithmeticRoundedSaturateScalar

* ShiftArithmeticRoundedScalar

* ShiftArithmeticSaturate

* ShiftArithmeticSaturateScalar

* ShiftArithmeticScalar

* ShiftLeftLogical

* ShiftLeftLogicalSaturate

* ShiftLeftLogicalSaturateScalar

* ShiftLeftLogicalSaturateUnsigned

* ShiftLeftLogicalSaturateUnsignedScalar

* ShiftLeftLogicalScalar

* ShiftLeftLogicalWideningLower

* ShiftLeftLogicalWideningUpper

* ShiftLogical

* ShiftLogicalRounded

* ShiftLogicalRoundedSaturate

* ShiftLogicalRoundedSaturateScalar

* ShiftLogicalRoundedScalar

* ShiftLogicalSaturate

* ShiftLogicalSaturateScalar

* ShiftLogicalScalar

* ShiftRightArithmetic

* ShiftRightArithmeticAdd

* ShiftRightArithmeticAddScalar

* ShiftRightArithmeticNarrowingSaturateLower

* ShiftRightArithmeticNarrowingSaturateUnsignedLower

* ShiftRightArithmeticNarrowingSaturateUnsignedUpper

* ShiftRightArithmeticNarrowingSaturateUpper

* ShiftRightArithmeticRounded

* ShiftRightArithmeticRoundedAdd

* ShiftRightArithmeticRoundedAddScalar

* ShiftRightArithmeticRoundedNarrowingSaturateLower

* ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower

* ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper

* ShiftRightArithmeticRoundedNarrowingSaturateUpper

* ShiftRightArithmeticRoundedScalar

* ShiftRightArithmeticScalar

* ShiftRightLogical

* ShiftRightLogicalAdd

* ShiftRightLogicalAddScalar

* ShiftRightLogicalNarrowingLower

* ShiftRightLogicalNarrowingSaturateLower

* ShiftRightLogicalNarrowingSaturateUpper

* ShiftRightLogicalNarrowingUpper

* ShiftRightLogicalRounded

* ShiftRightLogicalRoundedAdd

* ShiftRightLogicalRoundedAddScalar

* ShiftRightLogicalRoundedNarrowingLower

* ShiftRightLogicalRoundedNarrowingSaturateLower

* ShiftRightLogicalRoundedNarrowingSaturateUpper

* ShiftRightLogicalRoundedNarrowingUpper

* ShiftRightLogicalRoundedScalar

* ShiftRightLogicalScalar

* SignExtendWideningLower

* SignExtendWideningUpper

* ZeroExtendWideningLower

* ZeroExtendWideningUpper

* ShiftArithmeticRoundedSaturateScalar

* ShiftArithmeticSaturateScalar

* ShiftLeftLogicalSaturateScalar

* ShiftLeftLogicalSaturateUnsignedScalar

* ShiftLogicalRoundedSaturateScalar

* ShiftLogicalSaturateScalar

* ShiftRightArithmeticNarrowingSaturateScalar

* ShiftRightArithmeticNarrowingSaturateUnsignedScalar

* ShiftRightArithmeticRoundedNarrowingSaturateScalar

* ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar

* ShiftRightLogicalNarrowingSaturateScalar

* ShiftRightLogicalRoundedNarrowingSaturateScalar
  • Loading branch information
@echesakov
echesakov authored May 23, 2020
1 parent 75141e1 commit a26363e
Show file tree
Hide file tree
Showing 438 changed files with 200,480 additions and 1,875 deletions.
5 changes: 3 additions & 2 deletions src/coreclr/src/jit/codegen.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1060,7 +1060,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
// Returns true after the last call to EmitCaseEnd() (i.e. this signals that code generation is done).
bool Done() const
{
return immValue == immUpperBound;
return (immValue > immUpperBound);
}

// Returns a value of the immediate operand that should be used for a case.
Expand All @@ -1081,7 +1081,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
bool TestImmOpZeroOrOne() const
{
assert(NonConstImmOp());
return immUpperBound == 2;
return (immLowerBound == 0) && (immUpperBound == 1);
}

emitter* GetEmitter() const
Expand All @@ -1093,6 +1093,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
BasicBlock* endLabel;
BasicBlock* nonZeroLabel;
int immValue;
int immLowerBound;
int immUpperBound;
regNumber nonConstImmReg;
regNumber branchTargetReg;
Expand Down
3 changes: 2 additions & 1 deletion src/coreclr/src/jit/compiler.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3771,7 +3771,8 @@ class Compiler

GenTree* getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE argClass, bool expectAddr = false);
GenTree* impNonConstFallback(NamedIntrinsic intrinsic, var_types simdType, var_types baseType);
GenTree* addRangeCheckIfNeeded(NamedIntrinsic intrinsic, GenTree* lastOp, bool mustExpand, int immUpperBound);
GenTree* addRangeCheckIfNeeded(
NamedIntrinsic intrinsic, GenTree* lastOp, bool mustExpand, int immLowerBound, int immUpperBound);

#ifdef TARGET_XARCH
GenTree* impBaseIntrinsic(NamedIntrinsic intrinsic,
Expand Down
52 changes: 37 additions & 15 deletions src/coreclr/src/jit/hwintrinsic.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,13 +511,15 @@ GenTree* Compiler::getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE
// intrinsic -- intrinsic ID
// immOp -- the immediate operand of the intrinsic
// mustExpand -- true if the compiler is compiling the fallback(GT_CALL) of this intrinsics
// immUpperBound -- upper bound for a value of the immediate operand (for a non-full-range imm-intrinsic)
// immLowerBound -- lower incl. bound for a value of the immediate operand (for a non-full-range imm-intrinsic)
// immUpperBound -- upper incl. bound for a value of the immediate operand (for a non-full-range imm-intrinsic)
//
// Return Value:
// add a GT_HW_INTRINSIC_CHK node for non-full-range imm-intrinsic, which would throw ArgumentOutOfRangeException
// when the imm-argument is not in the valid range
//
GenTree* Compiler::addRangeCheckIfNeeded(NamedIntrinsic intrinsic, GenTree* immOp, bool mustExpand, int immUpperBound)
GenTree* Compiler::addRangeCheckIfNeeded(
NamedIntrinsic intrinsic, GenTree* immOp, bool mustExpand, int immLowerBound, int immUpperBound)
{
assert(immOp != nullptr);
// Full-range imm-intrinsics do not need the range-check
Expand All @@ -531,19 +533,37 @@ GenTree* Compiler::addRangeCheckIfNeeded(NamedIntrinsic intrinsic, GenTree* immO
)
{
assert(!immOp->IsCnsIntOrI());
GenTree* upperBoundNode = gtNewIconNode(immUpperBound, TYP_INT);
GenTree* index = nullptr;
if ((immOp->gtFlags & GTF_SIDE_EFFECT) != 0)
{
index = fgInsertCommaFormTemp(&immOp);
}
else
assert(varTypeIsUnsigned(immOp));

// Bounds check for value of an immediate operand
// (immLowerBound <= immOp) && (immOp <= immUpperBound)
//
// implemented as a single comparison in the form of
//
// if ((immOp - immLowerBound) >= (immUpperBound - immLowerBound + 1))
// {
// throw new ArgumentOutOfRangeException();
// }
//
// The value of (immUpperBound - immLowerBound + 1) is denoted as adjustedUpperBound.

const ssize_t adjustedUpperBound = (ssize_t)immUpperBound - immLowerBound + 1;
GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, TYP_INT);

GenTree* immOpDup = nullptr;

immOp = impCloneExpr(immOp, &immOpDup, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL,
nullptr DEBUGARG("Clone an immediate operand for immediate value bounds check"));

if (immLowerBound != 0)
{
index = gtCloneExpr(immOp);
immOpDup = gtNewOperNode(GT_SUB, TYP_INT, immOpDup, gtNewIconNode(immLowerBound, TYP_INT));
}

GenTreeBoundsChk* hwIntrinsicChk = new (this, GT_HW_INTRINSIC_CHK)
GenTreeBoundsChk(GT_HW_INTRINSIC_CHK, TYP_VOID, index, upperBoundNode, SCK_RNGCHK_FAIL);
GenTreeBoundsChk(GT_HW_INTRINSIC_CHK, TYP_VOID, immOpDup, adjustedUpperBoundNode, SCK_RNGCHK_FAIL);
hwIntrinsicChk->gtThrowKind = SCK_ARG_RNG_EXCPN;

return gtNewOperNode(GT_COMMA, immOp->TypeGet(), hwIntrinsicChk, immOp);
}
else
Expand Down Expand Up @@ -776,18 +796,20 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
var_types argType = TYP_UNKNOWN;
CORINFO_CLASS_HANDLE argClass;

int immLowerBound = 0;
int immUpperBound = 0;

if (immOp != nullptr)
{
#if defined(TARGET_XARCH)
immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(intrinsic);
#elif defined(TARGET_ARM64)
immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(intrinsic, simdSize, baseType);
HWIntrinsicInfo::lookupImmBounds(intrinsic, simdSize, baseType, &immLowerBound, &immUpperBound);
#endif
}

assert(numArgs >= 0);

if (!isScalar && ((HWIntrinsicInfo::lookupIns(intrinsic, baseType) == INS_invalid) ||
((simdSize != 8) && (simdSize != 16) && (simdSize != 32))))
{
Expand Down Expand Up @@ -835,7 +857,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
op2 = getArgForHWIntrinsic(argType, argClass);

op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand, immUpperBound);
op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand, immLowerBound, immUpperBound);

argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
op1 = getArgForHWIntrinsic(argType, argClass);
Expand Down Expand Up @@ -886,12 +908,12 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
#ifdef TARGET_ARM64
if (intrinsic == NI_AdvSimd_Insert)
{
op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand, immUpperBound);
op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand, immLowerBound, immUpperBound);
}
else
#endif
{
op3 = addRangeCheckIfNeeded(intrinsic, op3, mustExpand, immUpperBound);
op3 = addRangeCheckIfNeeded(intrinsic, op3, mustExpand, immLowerBound, immUpperBound);
}

retNode = isScalar ? gtNewScalarHWIntrinsicNode(retType, op1, op2, op3, intrinsic)
Expand Down
3 changes: 2 additions & 1 deletion src/coreclr/src/jit/hwintrinsic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,8 @@ struct HWIntrinsicInfo
#if defined(TARGET_XARCH)
static int lookupImmUpperBound(NamedIntrinsic intrinsic);
#elif defined(TARGET_ARM64)
static int lookupImmUpperBound(NamedIntrinsic intrinsic, int simdSize, var_types baseType);
static void lookupImmBounds(
NamedIntrinsic intrinsic, int simdSize, var_types baseType, int* lowerBound, int* upperBound);
#else
#error Unsupported platform
#endif
Expand Down
83 changes: 76 additions & 7 deletions src/coreclr/src/jit/hwintrinsicarm64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,19 +177,24 @@ bool HWIntrinsicInfo::isScalarIsa(CORINFO_InstructionSet isa)
}

//------------------------------------------------------------------------
// lookupImmUpperBound: Gets the upper bound for the imm-value of a given NamedIntrinsic
// lookupImmBounds: Gets the lower and upper bounds for the imm-value of a given NamedIntrinsic
//
// Arguments:
// intrinsic -- NamedIntrinsic associated with the HWIntrinsic to lookup
// simdType -- vector size
// baseType -- base type of the Vector64/128<T>
// pImmLowerBound [OUT] - The lower incl. bound for a value of the intrinsic immediate operand
// pImmUpperBound [OUT] - The upper incl. bound for a value of the intrinsic immediate operand
//
// Return Value:
// The upper bound for a value of the intrinsic immediate operand
int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic intrinsic, int simdSize, var_types baseType)
void HWIntrinsicInfo::lookupImmBounds(
NamedIntrinsic intrinsic, int simdSize, var_types baseType, int* pImmLowerBound, int* pImmUpperBound)
{
assert(HWIntrinsicInfo::lookupCategory(intrinsic) == HW_Category_IMM);

assert(pImmLowerBound != nullptr);
assert(pImmUpperBound != nullptr);

int immLowerBound = 0;
int immUpperBound = 0;

if (HWIntrinsicInfo::HasFullRangeImm(intrinsic))
Expand All @@ -209,15 +214,74 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic intrinsic, int simdSize,
case NI_AdvSimd_Arm64_DuplicateSelectedScalarToVector128:
case NI_Vector64_GetElement:
case NI_Vector128_GetElement:
immUpperBound = Compiler::getSIMDVectorLength(simdSize, baseType);
immUpperBound = Compiler::getSIMDVectorLength(simdSize, baseType) - 1;
break;

case NI_AdvSimd_ShiftLeftLogical:
case NI_AdvSimd_ShiftLeftLogicalSaturate:
case NI_AdvSimd_ShiftLeftLogicalSaturateScalar:
case NI_AdvSimd_ShiftLeftLogicalSaturateUnsigned:
case NI_AdvSimd_ShiftLeftLogicalSaturateUnsignedScalar:
case NI_AdvSimd_ShiftLeftLogicalScalar:
case NI_AdvSimd_ShiftLeftLogicalWideningLower:
case NI_AdvSimd_ShiftLeftLogicalWideningUpper:
case NI_AdvSimd_Arm64_ShiftLeftLogicalSaturateScalar:
case NI_AdvSimd_Arm64_ShiftLeftLogicalSaturateUnsignedScalar:
// The left shift amount is in the range 0 to the element width in bits minus 1.
immUpperBound = BITS_PER_BYTE * genTypeSize(baseType) - 1;
break;

case NI_AdvSimd_ShiftRightArithmetic:
case NI_AdvSimd_ShiftRightArithmeticAdd:
case NI_AdvSimd_ShiftRightArithmeticAddScalar:
case NI_AdvSimd_ShiftRightArithmeticNarrowingSaturateLower:
case NI_AdvSimd_ShiftRightArithmeticNarrowingSaturateUnsignedLower:
case NI_AdvSimd_ShiftRightArithmeticNarrowingSaturateUnsignedUpper:
case NI_AdvSimd_ShiftRightArithmeticNarrowingSaturateUpper:
case NI_AdvSimd_ShiftRightArithmeticRounded:
case NI_AdvSimd_ShiftRightArithmeticRoundedAdd:
case NI_AdvSimd_ShiftRightArithmeticRoundedAddScalar:
case NI_AdvSimd_ShiftRightArithmeticRoundedNarrowingSaturateLower:
case NI_AdvSimd_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower:
case NI_AdvSimd_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper:
case NI_AdvSimd_ShiftRightArithmeticRoundedNarrowingSaturateUpper:
case NI_AdvSimd_ShiftRightArithmeticRoundedScalar:
case NI_AdvSimd_ShiftRightArithmeticScalar:
case NI_AdvSimd_ShiftRightLogical:
case NI_AdvSimd_ShiftRightLogicalAdd:
case NI_AdvSimd_ShiftRightLogicalAddScalar:
case NI_AdvSimd_ShiftRightLogicalNarrowingLower:
case NI_AdvSimd_ShiftRightLogicalNarrowingSaturateLower:
case NI_AdvSimd_ShiftRightLogicalNarrowingSaturateUpper:
case NI_AdvSimd_ShiftRightLogicalNarrowingUpper:
case NI_AdvSimd_ShiftRightLogicalRounded:
case NI_AdvSimd_ShiftRightLogicalRoundedAdd:
case NI_AdvSimd_ShiftRightLogicalRoundedAddScalar:
case NI_AdvSimd_ShiftRightLogicalRoundedNarrowingLower:
case NI_AdvSimd_ShiftRightLogicalRoundedNarrowingSaturateLower:
case NI_AdvSimd_ShiftRightLogicalRoundedNarrowingSaturateUpper:
case NI_AdvSimd_ShiftRightLogicalRoundedNarrowingUpper:
case NI_AdvSimd_ShiftRightLogicalRoundedScalar:
case NI_AdvSimd_ShiftRightLogicalScalar:
case NI_AdvSimd_Arm64_ShiftRightArithmeticNarrowingSaturateScalar:
case NI_AdvSimd_Arm64_ShiftRightArithmeticNarrowingSaturateUnsignedScalar:
case NI_AdvSimd_Arm64_ShiftRightArithmeticRoundedNarrowingSaturateScalar:
case NI_AdvSimd_Arm64_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar:
case NI_AdvSimd_Arm64_ShiftRightLogicalNarrowingSaturateScalar:
case NI_AdvSimd_Arm64_ShiftRightLogicalRoundedNarrowingSaturateScalar:
// The right shift amount, in the range 1 to the element width in bits.
immLowerBound = 1;
immUpperBound = BITS_PER_BYTE * genTypeSize(baseType);
break;
default:
unreached();
}
}

return immUpperBound;
assert(immLowerBound <= immUpperBound);

*pImmLowerBound = immLowerBound;
*pImmUpperBound = immUpperBound;
}

//------------------------------------------------------------------------
Expand All @@ -236,7 +300,12 @@ bool HWIntrinsicInfo::isInImmRange(NamedIntrinsic id, int ival, int simdSize, va
{
assert(HWIntrinsicInfo::lookupCategory(id) == HW_Category_IMM);

return ival <= lookupImmUpperBound(id, simdSize, baseType) && ival >= 0;
int immLowerBound = 0;
int immUpperBound = 0;

lookupImmBounds(id, simdSize, baseType, &immLowerBound, &immUpperBound);

return (immLowerBound <= ival) && (ival <= immUpperBound);
}

//------------------------------------------------------------------------
Expand Down
Loading

0 comments on commit a26363e

Please sign in to comment.