Skip to content

Commit 4b0e2d0

Browse files
tannergoodingtannergooding
committed
Expose the x86 WaitPkg intrinsics
1 parent d4c1aa9 commit 4b0e2d0

File tree

23 files changed

+327
-94
lines changed

23 files changed

+327
-94
lines changed

‎src/coreclr/jit/codegen.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -940,13 +940,12 @@ class CodeGen final : public CodeGenInterface
940940
GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, int8_t ival, insOpts instOptions);
941941

942942
void genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
943-
void genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
943+
void genHWIntrinsicSpecial(GenTreeHWIntrinsic* node, insOpts instOptions);
944944
void genSse42Intrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
945945
void genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
946946
void genFmaIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
947947
void genPermuteVar2x(GenTreeHWIntrinsic* node, insOpts instOptions);
948948
void genXCNTIntrinsic(GenTreeHWIntrinsic* node, instruction ins);
949-
void genX86SerializeIntrinsic(GenTreeHWIntrinsic* node);
950949

951950
template <typename HWIntrinsicSwitchCaseBody>
952951
void genHWIntrinsicJumpTableFallback(NamedIntrinsic intrinsic,

‎src/coreclr/jit/fgdiagnostic.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3450,6 +3450,9 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block)
34503450
case NI_X86Base_Prefetch1:
34513451
case NI_X86Base_Prefetch2:
34523452
case NI_X86Base_PrefetchNonTemporal:
3453+
case NI_WAITPKG_SetUpUserLevelMonitor:
3454+
case NI_WAITPKG_TimedPause:
3455+
case NI_WAITPKG_WaitForUserLevelMonitor:
34533456
{
34543457
assert(tree->OperRequiresCallFlag(this));
34553458
expectedFlags |= GTF_GLOB_REF;

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29069,6 +29069,9 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const
2906929069
case NI_X86Base_Prefetch1:
2907029070
case NI_X86Base_Prefetch2:
2907129071
case NI_X86Base_PrefetchNonTemporal:
29072+
case NI_WAITPKG_SetUpUserLevelMonitor:
29073+
case NI_WAITPKG_TimedPause:
29074+
case NI_WAITPKG_WaitForUserLevelMonitor:
2907229075
{
2907329076
return true;
2907429077
}
@@ -29274,6 +29277,9 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId)
2927429277
case NI_X86Base_Prefetch1:
2927529278
case NI_X86Base_Prefetch2:
2927629279
case NI_X86Base_PrefetchNonTemporal:
29280+
case NI_WAITPKG_SetUpUserLevelMonitor:
29281+
case NI_WAITPKG_TimedPause:
29282+
case NI_WAITPKG_WaitForUserLevelMonitor:
2927729283
{
2927829284
// Mark as a call and global reference, much as is done for GT_KEEPALIVE
2927929285
gtFlags |= (GTF_CALL | GTF_GLOB_REF);

‎src/coreclr/jit/hwintrinsic.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = {
922922
{ FIRST_NI_GFNI_V256, LAST_NI_GFNI_V256 }, // GFNI_V256
923923
{ FIRST_NI_GFNI_V512, LAST_NI_GFNI_V512 }, // GFNI_V512
924924
{ NI_Illegal, NI_Illegal }, // SHA
925-
{ NI_Illegal, NI_Illegal }, // WAITPKG
925+
{ FIRST_NI_WAITPKG, LAST_NI_WAITPKG }, // WAITPKG
926926
{ FIRST_NI_X86Serialize, LAST_NI_X86Serialize }, // X86Serialize
927927
{ FIRST_NI_Vector128, LAST_NI_Vector128 }, // Vector128
928928
{ FIRST_NI_Vector256, LAST_NI_Vector256 }, // Vector256

‎src/coreclr/jit/hwintrinsiccodegenxarch.cpp‎

Lines changed: 50 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -998,10 +998,12 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
998998
break;
999999
}
10001000

1001+
case InstructionSet_WAITPKG:
10011002
case InstructionSet_X86Base:
10021003
case InstructionSet_X86Base_X64:
1004+
case InstructionSet_X86Serialize:
10031005
{
1004-
genX86BaseIntrinsic(node, instOptions);
1006+
genHWIntrinsicSpecial(node, instOptions);
10051007
break;
10061008
}
10071009

@@ -1025,14 +1027,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
10251027
break;
10261028
}
10271029

1028-
case InstructionSet_X86Serialize:
1029-
case InstructionSet_X86Serialize_X64:
1030-
{
1031-
assert(instOptions == INS_OPTS_NONE);
1032-
genX86SerializeIntrinsic(node);
1033-
break;
1034-
}
1035-
10361030
default:
10371031
unreached();
10381032
break;
@@ -2362,12 +2356,12 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
23622356
}
23632357

23642358
//------------------------------------------------------------------------
2365-
// genX86BaseIntrinsic: Generates the code for an X86 base hardware intrinsic node
2359+
// genHWIntrinsicSpecial: Generates the code for an special hardware intrinsic node
23662360
//
23672361
// Arguments:
23682362
// node - The hardware intrinsic node
23692363
//
2370-
void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
2364+
void CodeGen::genHWIntrinsicSpecial(GenTreeHWIntrinsic* node, insOpts instOptions)
23712365
{
23722366
NamedIntrinsic intrinsicId = node->GetHWIntrinsicId();
23732367
regNumber targetReg = node->GetRegNum();
@@ -2450,6 +2444,7 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
24502444
case NI_X86Base_Prefetch1:
24512445
case NI_X86Base_Prefetch2:
24522446
case NI_X86Base_PrefetchNonTemporal:
2447+
case NI_WAITPKG_SetUpUserLevelMonitor:
24532448
{
24542449
assert(baseType == TYP_UBYTE);
24552450
assert(instOptions == INS_OPTS_NONE);
@@ -2518,6 +2513,50 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
25182513
break;
25192514
}
25202515

2516+
case NI_X86Serialize_Serialize:
2517+
{
2518+
assert(instOptions == INS_OPTS_NONE);
2519+
assert(node->GetSimdBaseType() == TYP_UNKNOWN);
2520+
GetEmitter()->emitIns(INS_serialize);
2521+
break;
2522+
}
2523+
2524+
case NI_WAITPKG_TimedPause:
2525+
case NI_WAITPKG_WaitForUserLevelMonitor:
2526+
{
2527+
assert(node->GetSimdBaseType() == TYP_INT);
2528+
2529+
assert(node->GetOperandCount() == 3);
2530+
assert(instOptions == INS_OPTS_NONE);
2531+
2532+
GenTree* op1 = node->Op(1);
2533+
GenTree* op2 = node->Op(2);
2534+
GenTree* op3 = node->Op(3);
2535+
2536+
instruction ins = HWIntrinsicInfo::lookupIns(intrinsicId, TYP_INT, compiler);
2537+
2538+
regNumber op1Reg = op1->GetRegNum();
2539+
regNumber op2Reg = op2->GetRegNum();
2540+
regNumber op3Reg = op3->GetRegNum();
2541+
2542+
emitAttr attr = emitTypeSize(TYP_INT);
2543+
2544+
// op1: free, op2: EDX, op3: EAX
2545+
2546+
assert(op1Reg != REG_EDX);
2547+
assert(op1Reg != REG_EAX);
2548+
2549+
assert(op2Reg != REG_EAX);
2550+
assert(op3Reg != REG_EDX);
2551+
2552+
emit->emitIns_Mov(INS_mov, attr, REG_EDX, op2Reg, /* canSkip */ true);
2553+
emit->emitIns_Mov(INS_mov, attr, REG_EAX, op3Reg, /* canSkip */ true);
2554+
2555+
// emit the TPAUSE/UMWAIT instruction
2556+
emit->emitIns_R(ins, attr, op1Reg);
2557+
break;
2558+
}
2559+
25212560
default:
25222561
unreached();
25232562
break;
@@ -3916,33 +3955,4 @@ void CodeGen::genXCNTIntrinsic(GenTreeHWIntrinsic* node, instruction ins)
39163955
genHWIntrinsic_R_RM(node, ins, emitTypeSize(node->TypeGet()), targetReg, op1, INS_OPTS_NONE);
39173956
}
39183957

3919-
//------------------------------------------------------------------------
3920-
// genX86SerializeIntrinsic: Generates the code for an X86 serialize hardware intrinsic node
3921-
//
3922-
// Arguments:
3923-
// node - The hardware intrinsic node
3924-
//
3925-
void CodeGen::genX86SerializeIntrinsic(GenTreeHWIntrinsic* node)
3926-
{
3927-
NamedIntrinsic intrinsicId = node->GetHWIntrinsicId();
3928-
3929-
genConsumeMultiOpOperands(node);
3930-
3931-
switch (intrinsicId)
3932-
{
3933-
case NI_X86Serialize_Serialize:
3934-
{
3935-
assert(node->GetSimdBaseType() == TYP_UNKNOWN);
3936-
GetEmitter()->emitIns(INS_serialize);
3937-
break;
3938-
}
3939-
3940-
default:
3941-
unreached();
3942-
break;
3943-
}
3944-
3945-
genProduceReg(node);
3946-
}
3947-
39483958
#endif // FEATURE_HW_INTRINSICS

‎src/coreclr/jit/hwintrinsiclistxarch.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,6 +1202,17 @@ HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransformInverse,
12021202
HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, 64, 2, {INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
12031203
#define LAST_NI_GFNI_V512 NI_GFNI_V512_GaloisFieldMultiply
12041204

1205+
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
1206+
// ISA Function name SIMD size NumArg Instructions Category Flags
1207+
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}
1208+
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
1209+
// AVX512 Intrinsics for X86Serialize
1210+
#define FIRST_NI_WAITPKG NI_X86Serialize_Serialize
1211+
HARDWARE_INTRINSIC(WAITPKG, SetUpUserLevelMonitor, 0, 1, {INS_invalid, INS_umonitor, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other)
1212+
HARDWARE_INTRINSIC(WAITPKG, TimedPause, 0, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tpause, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other)
1213+
HARDWARE_INTRINSIC(WAITPKG, WaitForUserLevelMonitor, 0, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_umwait, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other)
1214+
#define LAST_NI_WAITPKG NI_X86Serialize_Serialize
1215+
12051216
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
12061217
// ISA Function name SIMD size NumArg Instructions Category Flags
12071218
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}

‎src/coreclr/jit/hwintrinsicxarch.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4275,6 +4275,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
42754275
case NI_X86Base_Prefetch1:
42764276
case NI_X86Base_Prefetch2:
42774277
case NI_X86Base_PrefetchNonTemporal:
4278+
case NI_WAITPKG_SetUpUserLevelMonitor:
42784279
{
42794280
assert(sig->numArgs == 1);
42804281
assert(JITtype2varType(sig->retType) == TYP_VOID);
@@ -4315,6 +4316,29 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
43154316
break;
43164317
}
43174318

4319+
case NI_WAITPKG_TimedPause:
4320+
case NI_WAITPKG_WaitForUserLevelMonitor:
4321+
{
4322+
assert(sig->numArgs == 2);
4323+
assert(JITtype2varType(sig->retType) == TYP_BYTE);
4324+
assert(simdSize == 0);
4325+
4326+
op2 = impPopStack().val;
4327+
op1 = impPopStack().val;
4328+
4329+
GenTree* op2Hi = nullptr;
4330+
GenTree* op2Lo = nullptr;
4331+
4332+
GenTree* op2Dup = fgMakeMultiUse(&op2);
4333+
4334+
op2Hi = gtFoldExpr(gtNewOperNode(GT_RSZ, TYP_LONG, op2, gtNewIconNode(32)));
4335+
op2Hi = gtFoldExpr(gtNewCastNode(TYP_INT, op2Hi, false, TYP_INT));
4336+
op2Lo = gtFoldExpr(gtNewCastNode(TYP_INT, op2Dup, false, TYP_INT));
4337+
4338+
retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2Hi, op2Lo, intrinsic, CORINFO_TYPE_INT, 0);
4339+
break;
4340+
}
4341+
43184342
case NI_AVX2_PermuteVar8x32:
43194343
case NI_AVX512_PermuteVar4x64:
43204344
case NI_AVX512_PermuteVar8x16:

‎src/coreclr/jit/lowerxarch.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2672,6 +2672,16 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node)
26722672
break;
26732673
}
26742674

2675+
case NI_WAITPKG_TimedPause:
2676+
case NI_WAITPKG_WaitForUserLevelMonitor:
2677+
{
2678+
GenTree* cc = LowerNodeCC(node, GenCondition::C);
2679+
2680+
node->gtType = TYP_VOID;
2681+
node->ClearUnusedValue();
2682+
break;
2683+
}
2684+
26752685
default:
26762686
break;
26772687
}

‎src/coreclr/jit/lsraxarch.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2858,6 +2858,21 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
28582858
break;
28592859
}
28602860

2861+
case NI_WAITPKG_TimedPause:
2862+
case NI_WAITPKG_WaitForUserLevelMonitor:
2863+
{
2864+
assert(numArgs == 3);
2865+
SingleTypeRegSet apxAwareRegCandidates =
2866+
ForceLowGprForApxIfNeeded(op1, RBM_NONE, canHWIntrinsicUseApxRegs);
2867+
2868+
srcCount += BuildOperandUses(op1, apxAwareRegCandidates);
2869+
srcCount += BuildOperandUses(op2, SRBM_EDX);
2870+
srcCount += BuildOperandUses(op3, SRBM_EAX);
2871+
2872+
buildUses = false;
2873+
break;
2874+
}
2875+
28612876
default:
28622877
{
28632878
assert((intrinsicId > NI_HW_INTRINSIC_START) && (intrinsicId < NI_HW_INTRINSIC_END));

‎src/libraries/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.NoX86Intrinsics.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,12 @@
174174
<type fullname="System.Runtime.Intrinsics.X86.Ssse3/X64">
175175
<method signature="System.Boolean get_IsSupported()" body="stub" value="false" />
176176
</type>
177+
<type fullname="System.Runtime.Intrinsics.X86.WaitPkg">
178+
<method signature="System.Boolean get_IsSupported()" body="stub" value="false" />
179+
</type>
180+
<type fullname="System.Runtime.Intrinsics.X86.WaitPkg/X64">
181+
<method signature="System.Boolean get_IsSupported()" body="stub" value="false" />
182+
</type>
177183
<type fullname="System.Runtime.Intrinsics.X86.X86Base">
178184
<method signature="System.Boolean get_IsSupported()" body="stub" value="false" />
179185
</type>

0 commit comments

Comments
 (0)