Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add GFNI Intrinsics #109537

Merged
merged 8 commits into from
Nov 22, 2024
Merged

Add GFNI Intrinsics #109537

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b1b8781
add GFNI intrinsics
saucecontrol Oct 26, 2024
f585454
add tests
saucecontrol Nov 1, 2024
e9f9bcf
rename file
saucecontrol Nov 5, 2024
0fe7832
Merge remote-tracking branch 'upstream/main' into gfni
saucecontrol Nov 20, 2024
b3b5b07
add missing tests and AOT handling
saucecontrol Nov 20, 2024
5d5d897
fix build
saucecontrol Nov 21, 2024
3e18b97
fix test result
saucecontrol Nov 21, 2024
ce21268
Merge remote-tracking branch 'upstream/main' into gfni
saucecontrol Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/coreclr/inc/clrconfigvalues.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,6 +778,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVXVNNI, W("EnableAVXVNNI
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableBMI1, W("EnableBMI1"), 1, "Allows BMI1+ hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableBMI2, W("EnableBMI2"), 1, "Allows BMI2+ hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableFMA, W("EnableFMA"), 1, "Allows FMA+ hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableGFNI, W("EnableGFNI"), 1, "Allows GFNI+ hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableLZCNT, W("EnableLZCNT"), 1, "Allows LZCNT+ hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnablePCLMULQDQ, W("EnablePCLMULQDQ"), 1, "Allows PCLMULQDQ+ hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableMOVBE, W("EnableMOVBE"), 1, "Allows MOVBE+ hardware intrinsics to be disabled")
Expand Down
212 changes: 128 additions & 84 deletions src/coreclr/inc/corinfoinstructionset.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,48 +66,51 @@ enum CORINFO_InstructionSet
InstructionSet_MOVBE=21,
InstructionSet_X86Serialize=22,
InstructionSet_EVEX=23,
InstructionSet_AVX512F=24,
InstructionSet_AVX512F_VL=25,
InstructionSet_AVX512BW=26,
InstructionSet_AVX512BW_VL=27,
InstructionSet_AVX512CD=28,
InstructionSet_AVX512CD_VL=29,
InstructionSet_AVX512DQ=30,
InstructionSet_AVX512DQ_VL=31,
InstructionSet_AVX512VBMI=32,
InstructionSet_AVX512VBMI_VL=33,
InstructionSet_AVX10v1=34,
InstructionSet_AVX10v1_V512=35,
InstructionSet_VectorT128=36,
InstructionSet_VectorT256=37,
InstructionSet_VectorT512=38,
InstructionSet_X86Base_X64=39,
InstructionSet_SSE_X64=40,
InstructionSet_SSE2_X64=41,
InstructionSet_SSE3_X64=42,
InstructionSet_SSSE3_X64=43,
InstructionSet_SSE41_X64=44,
InstructionSet_SSE42_X64=45,
InstructionSet_AVX_X64=46,
InstructionSet_AVX2_X64=47,
InstructionSet_AES_X64=48,
InstructionSet_BMI1_X64=49,
InstructionSet_BMI2_X64=50,
InstructionSet_FMA_X64=51,
InstructionSet_LZCNT_X64=52,
InstructionSet_PCLMULQDQ_X64=53,
InstructionSet_POPCNT_X64=54,
InstructionSet_AVXVNNI_X64=55,
InstructionSet_MOVBE_X64=56,
InstructionSet_X86Serialize_X64=57,
InstructionSet_EVEX_X64=58,
InstructionSet_AVX512F_X64=59,
InstructionSet_AVX512BW_X64=60,
InstructionSet_AVX512CD_X64=61,
InstructionSet_AVX512DQ_X64=62,
InstructionSet_AVX512VBMI_X64=63,
InstructionSet_AVX10v1_X64=64,
InstructionSet_AVX10v1_V512_X64=65,
InstructionSet_GFNI=24,
InstructionSet_GFNI_V256=25,
InstructionSet_GFNI_V512=26,
InstructionSet_AVX512F=27,
InstructionSet_AVX512F_VL=28,
InstructionSet_AVX512BW=29,
InstructionSet_AVX512BW_VL=30,
InstructionSet_AVX512CD=31,
InstructionSet_AVX512CD_VL=32,
InstructionSet_AVX512DQ=33,
InstructionSet_AVX512DQ_VL=34,
InstructionSet_AVX512VBMI=35,
InstructionSet_AVX512VBMI_VL=36,
InstructionSet_AVX10v1=37,
InstructionSet_AVX10v1_V512=38,
InstructionSet_VectorT128=39,
InstructionSet_VectorT256=40,
InstructionSet_VectorT512=41,
InstructionSet_X86Base_X64=42,
InstructionSet_SSE_X64=43,
InstructionSet_SSE2_X64=44,
InstructionSet_SSE3_X64=45,
InstructionSet_SSSE3_X64=46,
InstructionSet_SSE41_X64=47,
InstructionSet_SSE42_X64=48,
InstructionSet_AVX_X64=49,
InstructionSet_AVX2_X64=50,
InstructionSet_AES_X64=51,
InstructionSet_BMI1_X64=52,
InstructionSet_BMI2_X64=53,
InstructionSet_FMA_X64=54,
InstructionSet_LZCNT_X64=55,
InstructionSet_PCLMULQDQ_X64=56,
InstructionSet_POPCNT_X64=57,
InstructionSet_AVXVNNI_X64=58,
InstructionSet_MOVBE_X64=59,
InstructionSet_X86Serialize_X64=60,
InstructionSet_EVEX_X64=61,
InstructionSet_AVX512F_X64=62,
InstructionSet_AVX512BW_X64=63,
InstructionSet_AVX512CD_X64=64,
InstructionSet_AVX512DQ_X64=65,
InstructionSet_AVX512VBMI_X64=66,
InstructionSet_AVX10v1_X64=67,
InstructionSet_AVX10v1_V512_X64=68,
#endif // TARGET_AMD64
#ifdef TARGET_X86
InstructionSet_X86Base=1,
Expand All @@ -133,48 +136,51 @@ enum CORINFO_InstructionSet
InstructionSet_MOVBE=21,
InstructionSet_X86Serialize=22,
InstructionSet_EVEX=23,
InstructionSet_AVX512F=24,
InstructionSet_AVX512F_VL=25,
InstructionSet_AVX512BW=26,
InstructionSet_AVX512BW_VL=27,
InstructionSet_AVX512CD=28,
InstructionSet_AVX512CD_VL=29,
InstructionSet_AVX512DQ=30,
InstructionSet_AVX512DQ_VL=31,
InstructionSet_AVX512VBMI=32,
InstructionSet_AVX512VBMI_VL=33,
InstructionSet_AVX10v1=34,
InstructionSet_AVX10v1_V512=35,
InstructionSet_VectorT128=36,
InstructionSet_VectorT256=37,
InstructionSet_VectorT512=38,
InstructionSet_X86Base_X64=39,
InstructionSet_SSE_X64=40,
InstructionSet_SSE2_X64=41,
InstructionSet_SSE3_X64=42,
InstructionSet_SSSE3_X64=43,
InstructionSet_SSE41_X64=44,
InstructionSet_SSE42_X64=45,
InstructionSet_AVX_X64=46,
InstructionSet_AVX2_X64=47,
InstructionSet_AES_X64=48,
InstructionSet_BMI1_X64=49,
InstructionSet_BMI2_X64=50,
InstructionSet_FMA_X64=51,
InstructionSet_LZCNT_X64=52,
InstructionSet_PCLMULQDQ_X64=53,
InstructionSet_POPCNT_X64=54,
InstructionSet_AVXVNNI_X64=55,
InstructionSet_MOVBE_X64=56,
InstructionSet_X86Serialize_X64=57,
InstructionSet_EVEX_X64=58,
InstructionSet_AVX512F_X64=59,
InstructionSet_AVX512BW_X64=60,
InstructionSet_AVX512CD_X64=61,
InstructionSet_AVX512DQ_X64=62,
InstructionSet_AVX512VBMI_X64=63,
InstructionSet_AVX10v1_X64=64,
InstructionSet_AVX10v1_V512_X64=65,
InstructionSet_GFNI=24,
InstructionSet_GFNI_V256=25,
InstructionSet_GFNI_V512=26,
InstructionSet_AVX512F=27,
InstructionSet_AVX512F_VL=28,
InstructionSet_AVX512BW=29,
InstructionSet_AVX512BW_VL=30,
InstructionSet_AVX512CD=31,
InstructionSet_AVX512CD_VL=32,
InstructionSet_AVX512DQ=33,
InstructionSet_AVX512DQ_VL=34,
InstructionSet_AVX512VBMI=35,
InstructionSet_AVX512VBMI_VL=36,
InstructionSet_AVX10v1=37,
InstructionSet_AVX10v1_V512=38,
InstructionSet_VectorT128=39,
InstructionSet_VectorT256=40,
InstructionSet_VectorT512=41,
InstructionSet_X86Base_X64=42,
InstructionSet_SSE_X64=43,
InstructionSet_SSE2_X64=44,
InstructionSet_SSE3_X64=45,
InstructionSet_SSSE3_X64=46,
InstructionSet_SSE41_X64=47,
InstructionSet_SSE42_X64=48,
InstructionSet_AVX_X64=49,
InstructionSet_AVX2_X64=50,
InstructionSet_AES_X64=51,
InstructionSet_BMI1_X64=52,
InstructionSet_BMI2_X64=53,
InstructionSet_FMA_X64=54,
InstructionSet_LZCNT_X64=55,
InstructionSet_PCLMULQDQ_X64=56,
InstructionSet_POPCNT_X64=57,
InstructionSet_AVXVNNI_X64=58,
InstructionSet_MOVBE_X64=59,
InstructionSet_X86Serialize_X64=60,
InstructionSet_EVEX_X64=61,
InstructionSet_AVX512F_X64=62,
InstructionSet_AVX512BW_X64=63,
InstructionSet_AVX512CD_X64=64,
InstructionSet_AVX512DQ_X64=65,
InstructionSet_AVX512VBMI_X64=66,
InstructionSet_AVX10v1_X64=67,
InstructionSet_AVX10v1_V512_X64=68,
#endif // TARGET_X86

};
Expand Down Expand Up @@ -600,6 +606,16 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
resultflags.RemoveInstructionSet(InstructionSet_AVXVNNI);
if (resultflags.HasInstructionSet(InstructionSet_X86Serialize) && !resultflags.HasInstructionSet(InstructionSet_X86Base))
resultflags.RemoveInstructionSet(InstructionSet_X86Serialize);
if (resultflags.HasInstructionSet(InstructionSet_GFNI) && !resultflags.HasInstructionSet(InstructionSet_SSE41))
resultflags.RemoveInstructionSet(InstructionSet_GFNI);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V256) && !resultflags.HasInstructionSet(InstructionSet_GFNI))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V256);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V256) && !resultflags.HasInstructionSet(InstructionSet_AVX))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V256);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V512) && !resultflags.HasInstructionSet(InstructionSet_GFNI))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V512);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V512) && !resultflags.HasInstructionSet(InstructionSet_AVX512F))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V512);
if (resultflags.HasInstructionSet(InstructionSet_AVX10v1) && !resultflags.HasInstructionSet(InstructionSet_EVEX))
resultflags.RemoveInstructionSet(InstructionSet_AVX10v1);
if (resultflags.HasInstructionSet(InstructionSet_AVX10v1_V512) && !resultflags.HasInstructionSet(InstructionSet_AVX10v1))
Expand Down Expand Up @@ -706,6 +722,16 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
resultflags.RemoveInstructionSet(InstructionSet_AVXVNNI);
if (resultflags.HasInstructionSet(InstructionSet_X86Serialize) && !resultflags.HasInstructionSet(InstructionSet_X86Base))
resultflags.RemoveInstructionSet(InstructionSet_X86Serialize);
if (resultflags.HasInstructionSet(InstructionSet_GFNI) && !resultflags.HasInstructionSet(InstructionSet_SSE41))
resultflags.RemoveInstructionSet(InstructionSet_GFNI);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V256) && !resultflags.HasInstructionSet(InstructionSet_GFNI))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V256);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V256) && !resultflags.HasInstructionSet(InstructionSet_AVX))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V256);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V512) && !resultflags.HasInstructionSet(InstructionSet_GFNI))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V512);
if (resultflags.HasInstructionSet(InstructionSet_GFNI_V512) && !resultflags.HasInstructionSet(InstructionSet_AVX512F))
resultflags.RemoveInstructionSet(InstructionSet_GFNI_V512);
if (resultflags.HasInstructionSet(InstructionSet_AVX10v1) && !resultflags.HasInstructionSet(InstructionSet_EVEX))
resultflags.RemoveInstructionSet(InstructionSet_AVX10v1);
if (resultflags.HasInstructionSet(InstructionSet_AVX10v1_V512) && !resultflags.HasInstructionSet(InstructionSet_AVX10v1))
Expand Down Expand Up @@ -896,6 +922,12 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
return "EVEX";
case InstructionSet_EVEX_X64 :
return "EVEX_X64";
case InstructionSet_GFNI :
return "GFNI";
case InstructionSet_GFNI_V256 :
return "GFNI_V256";
case InstructionSet_GFNI_V512 :
return "GFNI_V512";
case InstructionSet_AVX512F :
return "AVX512F";
case InstructionSet_AVX512F_X64 :
Expand Down Expand Up @@ -988,6 +1020,12 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
return "X86Serialize";
case InstructionSet_EVEX :
return "EVEX";
case InstructionSet_GFNI :
return "GFNI";
case InstructionSet_GFNI_V256 :
return "GFNI_V256";
case InstructionSet_GFNI_V512 :
return "GFNI_V512";
case InstructionSet_AVX512F :
return "AVX512F";
case InstructionSet_AVX512F_VL :
Expand Down Expand Up @@ -1073,6 +1111,9 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
case READYTORUN_INSTRUCTION_Movbe: return InstructionSet_MOVBE;
case READYTORUN_INSTRUCTION_X86Serialize: return InstructionSet_X86Serialize;
case READYTORUN_INSTRUCTION_EVEX: return InstructionSet_EVEX;
case READYTORUN_INSTRUCTION_Gfni: return InstructionSet_GFNI;
case READYTORUN_INSTRUCTION_Gfni_V256: return InstructionSet_GFNI_V256;
case READYTORUN_INSTRUCTION_Gfni_V512: return InstructionSet_GFNI_V512;
case READYTORUN_INSTRUCTION_Avx512F: return InstructionSet_AVX512F;
case READYTORUN_INSTRUCTION_Avx512F_VL: return InstructionSet_AVX512F_VL;
case READYTORUN_INSTRUCTION_Avx512BW: return InstructionSet_AVX512BW;
Expand Down Expand Up @@ -1110,6 +1151,9 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
case READYTORUN_INSTRUCTION_Movbe: return InstructionSet_MOVBE;
case READYTORUN_INSTRUCTION_X86Serialize: return InstructionSet_X86Serialize;
case READYTORUN_INSTRUCTION_EVEX: return InstructionSet_EVEX;
case READYTORUN_INSTRUCTION_Gfni: return InstructionSet_GFNI;
case READYTORUN_INSTRUCTION_Gfni_V256: return InstructionSet_GFNI_V256;
case READYTORUN_INSTRUCTION_Gfni_V512: return InstructionSet_GFNI_V512;
case READYTORUN_INSTRUCTION_Avx512F: return InstructionSet_AVX512F;
case READYTORUN_INSTRUCTION_Avx512F_VL: return InstructionSet_AVX512F_VL;
case READYTORUN_INSTRUCTION_Avx512BW: return InstructionSet_AVX512BW;
Expand Down
10 changes: 5 additions & 5 deletions src/coreclr/inc/jiteeversionguid.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
#define GUID_DEFINED
#endif // !GUID_DEFINED

constexpr GUID JITEEVersionIdentifier = { /* 04021b93-e969-41ed-96cd-4c583673b9ab */
0x04021b93,
0xe969,
0x41ed,
{0x96, 0xcd, 0x4c, 0x58, 0x36, 0x73, 0xb9, 0xab}
constexpr GUID JITEEVersionIdentifier = { /* 64146448-11b1-4f94-b1f2-edce91fbcb33 */
0x64146448,
0x11b1,
0x4f94,
{0xb1, 0xf2, 0xed, 0xce, 0x91, 0xfb, 0xcb, 0x33}
};

//////////////////////////////////////////////////////////////////////////////////////////////////////////
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/inc/readytoruninstructionset.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ enum ReadyToRunInstructionSet
READYTORUN_INSTRUCTION_Avx10v1=44,
READYTORUN_INSTRUCTION_Avx10v1_V512=46,
READYTORUN_INSTRUCTION_EVEX=47,
READYTORUN_INSTRUCTION_Gfni=48,
READYTORUN_INSTRUCTION_Gfni_V256=49,
READYTORUN_INSTRUCTION_Gfni_V512=50,

};

Expand Down
7 changes: 7 additions & 0 deletions src/coreclr/jit/compiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6161,6 +6161,13 @@ int Compiler::compCompile(CORINFO_MODULE_HANDLE classPtr,
instructionSetFlags.AddInstructionSet(InstructionSet_FMA);
}

if (JitConfig.EnableGFNI() != 0)
{
instructionSetFlags.AddInstructionSet(InstructionSet_GFNI);
instructionSetFlags.AddInstructionSet(InstructionSet_GFNI_V256);
instructionSetFlags.AddInstructionSet(InstructionSet_GFNI_V512);
}

if (JitConfig.EnableLZCNT() != 0)
{
instructionSetFlags.AddInstructionSet(InstructionSet_LZCNT);
Expand Down
9 changes: 9 additions & 0 deletions src/coreclr/jit/emitxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1587,6 +1587,12 @@ bool emitter::TakesRexWPrefix(const instrDesc* id) const
return false;
}

case INS_gf2p8affineinvqb:
case INS_gf2p8affineqb:
{
return TakesVexPrefix(ins);
}

default:
{
unreached();
Expand Down Expand Up @@ -19824,6 +19830,9 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins
case INS_vpdpwssd:
case INS_vpdpbusds:
case INS_vpdpwssds:
case INS_gf2p8affineinvqb:
case INS_gf2p8affineqb:
case INS_gf2p8mulb:
result.insThroughput = PERFSCORE_THROUGHPUT_2X;
result.insLatency += PERFSCORE_LATENCY_5C;
break;
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/jit/hwintrinsic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,9 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = {
{ NI_Illegal, NI_Illegal }, // MOVBE
{ FIRST_NI_X86Serialize, LAST_NI_X86Serialize },
{ NI_Illegal, NI_Illegal }, // EVEX
{ FIRST_NI_GFNI, LAST_NI_GFNI },
{ FIRST_NI_GFNI_V256, LAST_NI_GFNI_V256 },
{ FIRST_NI_GFNI_V512, LAST_NI_GFNI_V512 },
{ FIRST_NI_AVX512F, LAST_NI_AVX512F },
{ FIRST_NI_AVX512F_VL, LAST_NI_AVX512F_VL },
{ FIRST_NI_AVX512BW, LAST_NI_AVX512BW },
Expand Down
Loading
Loading