Skip to content

Commit 58cc167

Browse files
nikicnikic
authored
[SDAG] Fix fmaximum legalization errors (#142170)
FMAXIMUM is currently legalized via IS_FPCLASS for the signed zero handling. This is problematic, because it assumes the equivalent integer type is legal. Many targets have legal fp128, but illegal i128, so this results in legalization failures. Fix this by replacing IS_FPCLASS with checking the bitcast to integer instead. In that case it is sufficient to use any legal integer type, as we're just interested in the sign bit. This can be obtained via a stack temporary cast. There is existing FloatSignAsInt functionality used for legalization of FABS and similar we can use for this purpose. Fixes #139380. Fixes #139381. Fixes #140445.
1 parent f3f07a3 commit 58cc167

File tree

12 files changed

+1183
-385
lines changed

12 files changed

+1183
-385
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAG.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,20 @@ class SDDbgInfo {
215215

216216
LLVM_ABI void checkForCycles(const SelectionDAG *DAG, bool force = false);
217217

218+
/// Keeps track of state when getting the sign of a floating-point value as an
219+
/// integer.
220+
struct FloatSignAsInt {
221+
EVT FloatVT;
222+
SDValue Chain;
223+
SDValue FloatPtr;
224+
SDValue IntPtr;
225+
MachinePointerInfo IntPointerInfo;
226+
MachinePointerInfo FloatPointerInfo;
227+
SDValue IntValue;
228+
APInt SignMask;
229+
uint8_t SignBit;
230+
};
231+
218232
/// This is used to represent a portion of an LLVM function in a low-level
219233
/// Data Dependence DAG representation suitable for instruction selection.
220234
/// This DAG is constructed as the first step of instruction selection in order
@@ -2017,6 +2031,16 @@ class SelectionDAG {
20172031
/// value types.
20182032
LLVM_ABI SDValue CreateStackTemporary(EVT VT1, EVT VT2);
20192033

2034+
/// Bitcast a floating-point value to an integer value. Only bitcast the part
2035+
/// containing the sign bit if the target has no integer value capable of
2036+
/// holding all bits of the floating-point value.
2037+
void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, SDValue Value);
2038+
2039+
/// Replace the integer value produced by getSignAsIntValue() with a new value
2040+
/// and cast the result back to a floating-point type.
2041+
SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
2042+
SDValue NewIntValue);
2043+
20202044
LLVM_ABI SDValue FoldSymbolOffset(unsigned Opcode, EVT VT,
20212045
const GlobalAddressSDNode *GA,
20222046
const SDNode *N2);

‎llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp‎

Lines changed: 7 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -59,20 +59,6 @@ using namespace llvm;
5959

6060
namespace {
6161

62-
/// Keeps track of state when getting the sign of a floating-point value as an
63-
/// integer.
64-
struct FloatSignAsInt {
65-
EVT FloatVT;
66-
SDValue Chain;
67-
SDValue FloatPtr;
68-
SDValue IntPtr;
69-
MachinePointerInfo IntPointerInfo;
70-
MachinePointerInfo FloatPointerInfo;
71-
SDValue IntValue;
72-
APInt SignMask;
73-
uint8_t SignBit;
74-
};
75-
7662
//===----------------------------------------------------------------------===//
7763
/// This takes an arbitrary SelectionDAG as input and
7864
/// hacks on it until the target machine can handle it. This involves
@@ -166,10 +152,6 @@ class SelectionDAGLegalize {
166152
SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
167153
void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
168154
SmallVectorImpl<SDValue> &Results);
169-
void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
170-
SDValue Value) const;
171-
SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
172-
SDValue NewIntValue) const;
173155
SDValue ExpandFCOPYSIGN(SDNode *Node) const;
174156
SDValue ExpandFABS(SDNode *Node) const;
175157
SDValue ExpandFNEG(SDNode *Node) const;
@@ -1620,82 +1602,14 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
16201602
return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
16211603
}
16221604

1623-
/// Bitcast a floating-point value to an integer value. Only bitcast the part
1624-
/// containing the sign bit if the target has no integer value capable of
1625-
/// holding all bits of the floating-point value.
1626-
void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1627-
const SDLoc &DL,
1628-
SDValue Value) const {
1629-
EVT FloatVT = Value.getValueType();
1630-
unsigned NumBits = FloatVT.getScalarSizeInBits();
1631-
State.FloatVT = FloatVT;
1632-
EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1633-
// Convert to an integer of the same size.
1634-
if (TLI.isTypeLegal(IVT)) {
1635-
State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1636-
State.SignMask = APInt::getSignMask(NumBits);
1637-
State.SignBit = NumBits - 1;
1638-
return;
1639-
}
1640-
1641-
auto &DataLayout = DAG.getDataLayout();
1642-
// Store the float to memory, then load the sign part out as an integer.
1643-
MVT LoadTy = TLI.getRegisterType(MVT::i8);
1644-
// First create a temporary that is aligned for both the load and store.
1645-
SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1646-
int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1647-
// Then store the float to it.
1648-
State.FloatPtr = StackPtr;
1649-
MachineFunction &MF = DAG.getMachineFunction();
1650-
State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1651-
State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1652-
State.FloatPointerInfo);
1653-
1654-
SDValue IntPtr;
1655-
if (DataLayout.isBigEndian()) {
1656-
assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1657-
// Load out a legal integer with the same sign bit as the float.
1658-
IntPtr = StackPtr;
1659-
State.IntPointerInfo = State.FloatPointerInfo;
1660-
} else {
1661-
// Advance the pointer so that the loaded byte will contain the sign bit.
1662-
unsigned ByteOffset = (NumBits / 8) - 1;
1663-
IntPtr =
1664-
DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1665-
State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1666-
ByteOffset);
1667-
}
1668-
1669-
State.IntPtr = IntPtr;
1670-
State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1671-
State.IntPointerInfo, MVT::i8);
1672-
State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1673-
State.SignBit = 7;
1674-
}
1675-
1676-
/// Replace the integer value produced by getSignAsIntValue() with a new value
1677-
/// and cast the result back to a floating-point type.
1678-
SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1679-
const SDLoc &DL,
1680-
SDValue NewIntValue) const {
1681-
if (!State.Chain)
1682-
return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1683-
1684-
// Override the part containing the sign bit in the value stored on the stack.
1685-
SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1686-
State.IntPointerInfo, MVT::i8);
1687-
return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1688-
State.FloatPointerInfo);
1689-
}
1690-
16911605
SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
16921606
SDLoc DL(Node);
16931607
SDValue Mag = Node->getOperand(0);
16941608
SDValue Sign = Node->getOperand(1);
16951609

16961610
// Get sign bit into an integer value.
16971611
FloatSignAsInt SignAsInt;
1698-
getSignAsIntValue(SignAsInt, DL, Sign);
1612+
DAG.getSignAsIntValue(SignAsInt, DL, Sign);
16991613

17001614
EVT IntVT = SignAsInt.IntValue.getValueType();
17011615
SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
@@ -1716,7 +1630,7 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
17161630

17171631
// Transform Mag value to integer, and clear the sign bit.
17181632
FloatSignAsInt MagAsInt;
1719-
getSignAsIntValue(MagAsInt, DL, Mag);
1633+
DAG.getSignAsIntValue(MagAsInt, DL, Mag);
17201634
EVT MagVT = MagAsInt.IntValue.getValueType();
17211635
SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
17221636
SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
@@ -1746,14 +1660,14 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
17461660
SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit,
17471661
SDNodeFlags::Disjoint);
17481662

1749-
return modifySignAsInt(MagAsInt, DL, CopiedSign);
1663+
return DAG.modifySignAsInt(MagAsInt, DL, CopiedSign);
17501664
}
17511665

17521666
SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
17531667
// Get the sign bit as an integer.
17541668
SDLoc DL(Node);
17551669
FloatSignAsInt SignAsInt;
1756-
getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1670+
DAG.getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
17571671
EVT IntVT = SignAsInt.IntValue.getValueType();
17581672

17591673
// Flip the sign.
@@ -1762,7 +1676,7 @@ SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
17621676
DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
17631677

17641678
// Convert back to float.
1765-
return modifySignAsInt(SignAsInt, DL, SignFlip);
1679+
return DAG.modifySignAsInt(SignAsInt, DL, SignFlip);
17661680
}
17671681

17681682
SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
@@ -1778,12 +1692,12 @@ SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
17781692

17791693
// Transform value to integer, clear the sign bit and transform back.
17801694
FloatSignAsInt ValueAsInt;
1781-
getSignAsIntValue(ValueAsInt, DL, Value);
1695+
DAG.getSignAsIntValue(ValueAsInt, DL, Value);
17821696
EVT IntVT = ValueAsInt.IntValue.getValueType();
17831697
SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
17841698
SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
17851699
ClearSignMask);
1786-
return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1700+
return DAG.modifySignAsInt(ValueAsInt, DL, ClearedSign);
17871701
}
17881702

17891703
void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2760,6 +2760,66 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
27602760
return CreateStackTemporary(Bytes, Align);
27612761
}
27622762

2763+
void SelectionDAG::getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
2764+
SDValue Value) {
2765+
EVT FloatVT = Value.getValueType();
2766+
unsigned NumBits = FloatVT.getScalarSizeInBits();
2767+
State.FloatVT = FloatVT;
2768+
EVT IVT = FloatVT.changeTypeToInteger();
2769+
// Convert to an integer of the same size.
2770+
if (TLI->isTypeLegal(IVT)) {
2771+
State.IntValue = getNode(ISD::BITCAST, DL, IVT, Value);
2772+
State.SignMask = APInt::getSignMask(NumBits);
2773+
State.SignBit = NumBits - 1;
2774+
return;
2775+
}
2776+
2777+
auto &DataLayout = getDataLayout();
2778+
// Store the float to memory, then load the sign part out as an integer.
2779+
MVT LoadTy = TLI->getRegisterType(MVT::i8);
2780+
// First create a temporary that is aligned for both the load and store.
2781+
SDValue StackPtr = CreateStackTemporary(FloatVT, LoadTy);
2782+
int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2783+
// Then store the float to it.
2784+
State.FloatPtr = StackPtr;
2785+
MachineFunction &MF = getMachineFunction();
2786+
State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
2787+
State.Chain = getStore(getEntryNode(), DL, Value, State.FloatPtr,
2788+
State.FloatPointerInfo);
2789+
2790+
SDValue IntPtr;
2791+
if (DataLayout.isBigEndian()) {
2792+
assert(FloatVT.isByteSized() && "Unsupported floating point type!");
2793+
// Load out a legal integer with the same sign bit as the float.
2794+
IntPtr = StackPtr;
2795+
State.IntPointerInfo = State.FloatPointerInfo;
2796+
} else {
2797+
// Advance the pointer so that the loaded byte will contain the sign bit.
2798+
unsigned ByteOffset = (NumBits / 8) - 1;
2799+
IntPtr = getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
2800+
State.IntPointerInfo =
2801+
MachinePointerInfo::getFixedStack(MF, FI, ByteOffset);
2802+
}
2803+
2804+
State.IntPtr = IntPtr;
2805+
State.IntValue = getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
2806+
State.IntPointerInfo, MVT::i8);
2807+
State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
2808+
State.SignBit = 7;
2809+
}
2810+
2811+
SDValue SelectionDAG::modifySignAsInt(const FloatSignAsInt &State,
2812+
const SDLoc &DL, SDValue NewIntValue) {
2813+
if (!State.Chain)
2814+
return getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
2815+
2816+
// Override the part containing the sign bit in the value stored on the stack.
2817+
SDValue Chain = getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
2818+
State.IntPointerInfo, MVT::i8);
2819+
return getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
2820+
State.FloatPointerInfo);
2821+
}
2822+
27632823
SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
27642824
ISD::CondCode Cond, const SDLoc &dl) {
27652825
EVT OpVT = N1.getValueType();

‎llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8610,16 +8610,18 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
86108610
// fminimum/fmaximum requires -0.0 less than +0.0
86118611
if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
86128612
!DAG.isKnownNeverZeroFloat(RHS) && !DAG.isKnownNeverZeroFloat(LHS)) {
8613+
auto IsSpecificZero = [&](SDValue F) {
8614+
FloatSignAsInt State;
8615+
DAG.getSignAsIntValue(State, DL, F);
8616+
return DAG.getSetCC(DL, CCVT, State.IntValue,
8617+
DAG.getConstant(0, DL, State.IntValue.getValueType()),
8618+
IsMax ? ISD::SETEQ : ISD::SETNE);
8619+
};
86138620
SDValue IsZero = DAG.getSetCC(DL, CCVT, MinMax,
86148621
DAG.getConstantFP(0.0, DL, VT), ISD::SETOEQ);
8615-
SDValue TestZero =
8616-
DAG.getTargetConstant(IsMax ? fcPosZero : fcNegZero, DL, MVT::i32);
8617-
SDValue LCmp = DAG.getSelect(
8618-
DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, LHS, TestZero), LHS,
8619-
MinMax, Flags);
8620-
SDValue RCmp = DAG.getSelect(
8621-
DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, RHS, TestZero), RHS,
8622-
LCmp, Flags);
8622+
SDValue LCmp =
8623+
DAG.getSelect(DL, VT, IsSpecificZero(LHS), LHS, MinMax, Flags);
8624+
SDValue RCmp = DAG.getSelect(DL, VT, IsSpecificZero(RHS), RHS, LCmp, Flags);
86238625
MinMax = DAG.getSelect(DL, VT, IsZero, RCmp, MinMax, Flags);
86248626
}
86258627

‎llvm/test/CodeGen/AArch64/fmaximum-legalization.ll‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,59 @@ define <4 x half> @fmaximum_v4f16(<4 x half> %x, <4 x half> %y) {
4141
%r = call <4 x half> @llvm.maximum.v4f16(<4 x half> %x, <4 x half> %y)
4242
ret <4 x half> %r
4343
}
44+
45+
define fp128 @maximum_fp128(fp128 %x, fp128 %y) nounwind {
46+
; CHECK-LABEL: maximum_fp128:
47+
; CHECK: // %bb.0:
48+
; CHECK-NEXT: sub sp, sp, #96
49+
; CHECK-NEXT: str x30, [sp, #80] // 8-byte Folded Spill
50+
; CHECK-NEXT: stp q0, q1, [sp] // 32-byte Folded Spill
51+
; CHECK-NEXT: stp q1, q0, [sp, #48]
52+
; CHECK-NEXT: bl __gttf2
53+
; CHECK-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
54+
; CHECK-NEXT: cmp w0, #0
55+
; CHECK-NEXT: b.le .LBB1_2
56+
; CHECK-NEXT: // %bb.1:
57+
; CHECK-NEXT: mov v1.16b, v0.16b
58+
; CHECK-NEXT: .LBB1_2:
59+
; CHECK-NEXT: str q1, [sp, #32] // 16-byte Folded Spill
60+
; CHECK-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
61+
; CHECK-NEXT: bl __unordtf2
62+
; CHECK-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
63+
; CHECK-NEXT: cmp w0, #0
64+
; CHECK-NEXT: b.eq .LBB1_4
65+
; CHECK-NEXT: // %bb.3:
66+
; CHECK-NEXT: adrp x8, .LCPI1_0
67+
; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI1_0]
68+
; CHECK-NEXT: .LBB1_4:
69+
; CHECK-NEXT: ldrb w8, [sp, #79]
70+
; CHECK-NEXT: mov v1.16b, v0.16b
71+
; CHECK-NEXT: cmp w8, #0
72+
; CHECK-NEXT: b.ne .LBB1_6
73+
; CHECK-NEXT: // %bb.5:
74+
; CHECK-NEXT: ldr q1, [sp] // 16-byte Folded Reload
75+
; CHECK-NEXT: .LBB1_6:
76+
; CHECK-NEXT: ldrb w8, [sp, #63]
77+
; CHECK-NEXT: cmp w8, #0
78+
; CHECK-NEXT: b.ne .LBB1_8
79+
; CHECK-NEXT: // %bb.7:
80+
; CHECK-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
81+
; CHECK-NEXT: .LBB1_8:
82+
; CHECK-NEXT: adrp x8, .LCPI1_1
83+
; CHECK-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
84+
; CHECK-NEXT: str q1, [sp, #16] // 16-byte Folded Spill
85+
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_1]
86+
; CHECK-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
87+
; CHECK-NEXT: bl __eqtf2
88+
; CHECK-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
89+
; CHECK-NEXT: cmp w0, #0
90+
; CHECK-NEXT: b.ne .LBB1_10
91+
; CHECK-NEXT: // %bb.9:
92+
; CHECK-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload
93+
; CHECK-NEXT: .LBB1_10:
94+
; CHECK-NEXT: ldr x30, [sp, #80] // 8-byte Folded Reload
95+
; CHECK-NEXT: add sp, sp, #96
96+
; CHECK-NEXT: ret
97+
%res = call fp128 @llvm.maximum.f128(fp128 %x, fp128 %y)
98+
ret fp128 %res
99+
}

0 commit comments

Comments
 (0)