Skip to content

Commit a847ecf

Browse files
jofrnjofrn
committed
[SelectionDAG][X86] Widen <2 x T> vector types for atomic load
Vector types of 2 elements must be widened. This change does this for vector types of atomic load in SelectionDAG so that it can translate aligned vectors of >1 size. Also, it also adds Pats to remove an extra MOV. commit-id:2894ccd1
1 parent 2af4d7c commit a847ecf

File tree

5 files changed

+124
-8
lines changed

5 files changed

+124
-8
lines changed

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -1048,6 +1048,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
10481048
SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
10491049
SDValue WidenVecRes_INSERT_SUBVECTOR(SDNode *N);
10501050
SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
1051+
SDValue WidenVecRes_ATOMIC_LOAD(AtomicSDNode *N);
10511052
SDValue WidenVecRes_LOAD(SDNode* N);
10521053
SDValue WidenVecRes_VP_LOAD(VPLoadSDNode *N);
10531054
SDValue WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N);
@@ -1131,8 +1132,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
11311132
/// resulting wider type. It takes:
11321133
/// LdChain: list of chains for the load to be generated.
11331134
/// Ld: load to widen
1134-
SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
1135-
LoadSDNode *LD);
1135+
SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, MemSDNode *LD,
1136+
bool IsAtomic = false);
11361137

11371138
/// Helper function to generate a set of extension loads to load a vector with
11381139
/// a resulting wider type. It takes:

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

+32-4
Original file line numberDiff line numberDiff line change
@@ -4521,6 +4521,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
45214521
break;
45224522
case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
45234523
case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4524+
case ISD::ATOMIC_LOAD:
4525+
Res = WidenVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
4526+
break;
45244527
case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
45254528
case ISD::STEP_VECTOR:
45264529
case ISD::SPLAT_VECTOR:
@@ -5907,6 +5910,26 @@ SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
59075910
N->getOperand(1), N->getOperand(2));
59085911
}
59095912

5913+
SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
5914+
SmallVector<SDValue, 16> LdChain; // Chain for the series of load
5915+
SDValue Result = GenWidenVectorLoads(LdChain, N, /*IsAtomic=*/true);
5916+
5917+
if (Result) {
5918+
// Build a factor node to remember the multiple loads are independent and
5919+
// chain to that.
5920+
SDValue NewChain =
5921+
DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, LdChain);
5922+
5923+
// Modified the chain - switch anything that used the old chain to use
5924+
// the new one.
5925+
ReplaceValueWith(SDValue(N, 1), NewChain);
5926+
5927+
return Result;
5928+
}
5929+
5930+
report_fatal_error("Unable to widen atomic vector load");
5931+
}
5932+
59105933
SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
59115934
LoadSDNode *LD = cast<LoadSDNode>(N);
59125935
ISD::LoadExtType ExtType = LD->getExtensionType();
@@ -7706,7 +7729,7 @@ static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
77067729
}
77077730

77087731
SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
7709-
LoadSDNode *LD) {
7732+
MemSDNode *LD, bool IsAtomic) {
77107733
// The strategy assumes that we can efficiently load power-of-two widths.
77117734
// The routine chops the vector into the largest vector loads with the same
77127735
// element type or scalar loads and then recombines it to the widen vector
@@ -7763,12 +7786,17 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
77637786
} while (TypeSize::isKnownGT(RemainingWidth, NewVTWidth));
77647787
}
77657788

7766-
SDValue LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr, LD->getPointerInfo(),
7767-
LD->getOriginalAlign(), MMOFlags, AAInfo);
7789+
SDValue LdOp;
7790+
if (IsAtomic)
7791+
LdOp = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, *FirstVT, *FirstVT, Chain,
7792+
BasePtr, LD->getMemOperand());
7793+
else
7794+
LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr, LD->getPointerInfo(),
7795+
LD->getOriginalAlign(), MMOFlags, AAInfo);
77687796
LdChain.push_back(LdOp.getValue(1));
77697797

77707798
// Check if we can load the element with one instruction.
7771-
if (MemVTs.empty()) {
7799+
if (MemVTs.empty() || IsAtomic) {
77727800
assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
77737801
if (!FirstVT->isVector()) {
77747802
unsigned NumElts =

‎llvm/lib/Target/X86/X86InstrCompiler.td

+7
Original file line numberDiff line numberDiff line change
@@ -1198,6 +1198,13 @@ def : Pat<(i16 (atomic_load_16 addr:$src)), (MOV16rm addr:$src)>;
11981198
def : Pat<(i32 (atomic_load_32 addr:$src)), (MOV32rm addr:$src)>;
11991199
def : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>;
12001200

1201+
def : Pat<(v4i32 (scalar_to_vector (i32 (anyext (i16 (atomic_load_16 addr:$src)))))),
1202+
(MOVDI2PDIrm addr:$src)>; // load atomic <2 x i8>
1203+
def : Pat<(v4i32 (scalar_to_vector (i32 (atomic_load_32 addr:$src)))),
1204+
(MOVDI2PDIrm addr:$src)>; // load atomic <2 x i16>
1205+
def : Pat<(v2i64 (scalar_to_vector (i64 (atomic_load_64 addr:$src)))),
1206+
(MOV64toPQIrm addr:$src)>; // load atomic <2 x i32,float>
1207+
12011208
// Floating point loads/stores.
12021209
def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst),
12031210
(MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>;

‎llvm/test/CodeGen/X86/atomic-load-store.ll

+81
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,64 @@ define <1 x i64> @atomic_vec1_i64_align(ptr %x) nounwind {
146146
ret <1 x i64> %ret
147147
}
148148

149+
define <2 x i8> @atomic_vec2_i8(ptr %x) {
150+
; CHECK3-LABEL: atomic_vec2_i8:
151+
; CHECK3: ## %bb.0:
152+
; CHECK3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
153+
; CHECK3-NEXT: retq
154+
;
155+
; CHECK0-LABEL: atomic_vec2_i8:
156+
; CHECK0: ## %bb.0:
157+
; CHECK0-NEXT: movw (%rdi), %cx
158+
; CHECK0-NEXT: ## implicit-def: $eax
159+
; CHECK0-NEXT: movw %cx, %ax
160+
; CHECK0-NEXT: movd %eax, %xmm0
161+
; CHECK0-NEXT: retq
162+
%ret = load atomic <2 x i8>, ptr %x acquire, align 4
163+
ret <2 x i8> %ret
164+
}
165+
166+
define <2 x i16> @atomic_vec2_i16(ptr %x) {
167+
; CHECK3-LABEL: atomic_vec2_i16:
168+
; CHECK3: ## %bb.0:
169+
; CHECK3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
170+
; CHECK3-NEXT: retq
171+
;
172+
; CHECK0-LABEL: atomic_vec2_i16:
173+
; CHECK0: ## %bb.0:
174+
; CHECK0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
175+
; CHECK0-NEXT: retq
176+
%ret = load atomic <2 x i16>, ptr %x acquire, align 4
177+
ret <2 x i16> %ret
178+
}
179+
180+
define <2 x ptr addrspace(270)> @atomic_vec2_ptr270(ptr %x) {
181+
; CHECK-LABEL: atomic_vec2_ptr270:
182+
; CHECK: ## %bb.0:
183+
; CHECK-NEXT: movq (%rdi), %xmm0
184+
; CHECK-NEXT: retq
185+
%ret = load atomic <2 x ptr addrspace(270)>, ptr %x acquire, align 8
186+
ret <2 x ptr addrspace(270)> %ret
187+
}
188+
189+
define <2 x i32> @atomic_vec2_i32_align(ptr %x) {
190+
; CHECK-LABEL: atomic_vec2_i32_align:
191+
; CHECK: ## %bb.0:
192+
; CHECK-NEXT: movq (%rdi), %xmm0
193+
; CHECK-NEXT: retq
194+
%ret = load atomic <2 x i32>, ptr %x acquire, align 8
195+
ret <2 x i32> %ret
196+
}
197+
198+
define <2 x float> @atomic_vec2_float_align(ptr %x) {
199+
; CHECK-LABEL: atomic_vec2_float_align:
200+
; CHECK: ## %bb.0:
201+
; CHECK-NEXT: movq (%rdi), %xmm0
202+
; CHECK-NEXT: retq
203+
%ret = load atomic <2 x float>, ptr %x acquire, align 8
204+
ret <2 x float> %ret
205+
}
206+
149207
define <1 x ptr> @atomic_vec1_ptr(ptr %x) nounwind {
150208
; CHECK3-LABEL: atomic_vec1_ptr:
151209
; CHECK3: ## %bb.0:
@@ -295,6 +353,29 @@ define <2 x i32> @atomic_vec2_i32(ptr %x) nounwind {
295353
ret <2 x i32> %ret
296354
}
297355

356+
define <4 x i8> @atomic_vec4_i8(ptr %x) nounwind {
357+
; CHECK3-LABEL: atomic_vec4_i8:
358+
; CHECK3: ## %bb.0:
359+
; CHECK3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
360+
; CHECK3-NEXT: retq
361+
;
362+
; CHECK0-LABEL: atomic_vec4_i8:
363+
; CHECK0: ## %bb.0:
364+
; CHECK0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
365+
; CHECK0-NEXT: retq
366+
%ret = load atomic <4 x i8>, ptr %x acquire, align 4
367+
ret <4 x i8> %ret
368+
}
369+
370+
define <4 x i16> @atomic_vec4_i16(ptr %x) nounwind {
371+
; CHECK-LABEL: atomic_vec4_i16:
372+
; CHECK: ## %bb.0:
373+
; CHECK-NEXT: movq (%rdi), %xmm0
374+
; CHECK-NEXT: retq
375+
%ret = load atomic <4 x i16>, ptr %x acquire, align 8
376+
ret <4 x i16> %ret
377+
}
378+
298379
define <4 x float> @atomic_vec4_float_align(ptr %x) nounwind {
299380
; CHECK-LABEL: atomic_vec4_float_align:
300381
; CHECK: ## %bb.0:

‎llvm/test/CodeGen/X86/atomic-unordered.ll

+1-2
Original file line numberDiff line numberDiff line change
@@ -2275,8 +2275,7 @@ define i64 @load_i16_anyext_i64(ptr %ptr) {
22752275
;
22762276
; CHECK-O3-LABEL: load_i16_anyext_i64:
22772277
; CHECK-O3: # %bb.0:
2278-
; CHECK-O3-NEXT: movzwl (%rdi), %eax
2279-
; CHECK-O3-NEXT: vmovd %eax, %xmm0
2278+
; CHECK-O3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
22802279
; CHECK-O3-NEXT: vmovq %xmm0, %rax
22812280
; CHECK-O3-NEXT: retq
22822281
%v = load atomic i16, ptr %ptr unordered, align 8

0 commit comments

Comments
 (0)