1 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that ARM uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "ARMISelLowering.h"
15 #include "ARMBaseInstrInfo.h"
16 #include "ARMBaseRegisterInfo.h"
17 #include "ARMCallingConv.h"
18 #include "ARMConstantPoolValue.h"
19 #include "ARMMachineFunctionInfo.h"
20 #include "ARMPerfectShuffle.h"
21 #include "ARMRegisterInfo.h"
22 #include "ARMSelectionDAGInfo.h"
23 #include "ARMSubtarget.h"
24 #include "ARMTargetTransformInfo.h"
25 #include "MCTargetDesc/ARMAddressingModes.h"
26 #include "MCTargetDesc/ARMBaseInfo.h"
27 #include "Utils/ARMBaseInfo.h"
28 #include "llvm/ADT/APFloat.h"
29 #include "llvm/ADT/APInt.h"
30 #include "llvm/ADT/ArrayRef.h"
31 #include "llvm/ADT/BitVector.h"
32 #include "llvm/ADT/DenseMap.h"
33 #include "llvm/ADT/STLExtras.h"
34 #include "llvm/ADT/SmallPtrSet.h"
35 #include "llvm/ADT/SmallVector.h"
36 #include "llvm/ADT/Statistic.h"
37 #include "llvm/ADT/StringExtras.h"
38 #include "llvm/ADT/StringRef.h"
39 #include "llvm/ADT/StringSwitch.h"
40 #include "llvm/ADT/Triple.h"
41 #include "llvm/ADT/Twine.h"
42 #include "llvm/Analysis/VectorUtils.h"
43 #include "llvm/CodeGen/CallingConvLower.h"
44 #include "llvm/CodeGen/ISDOpcodes.h"
45 #include "llvm/CodeGen/IntrinsicLowering.h"
46 #include "llvm/CodeGen/MachineBasicBlock.h"
47 #include "llvm/CodeGen/MachineConstantPool.h"
48 #include "llvm/CodeGen/MachineFrameInfo.h"
49 #include "llvm/CodeGen/MachineFunction.h"
50 #include "llvm/CodeGen/MachineInstr.h"
51 #include "llvm/CodeGen/MachineInstrBuilder.h"
52 #include "llvm/CodeGen/MachineJumpTableInfo.h"
53 #include "llvm/CodeGen/MachineMemOperand.h"
54 #include "llvm/CodeGen/MachineOperand.h"
55 #include "llvm/CodeGen/MachineRegisterInfo.h"
56 #include "llvm/CodeGen/RuntimeLibcalls.h"
57 #include "llvm/CodeGen/SelectionDAG.h"
58 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
59 #include "llvm/CodeGen/SelectionDAGNodes.h"
60 #include "llvm/CodeGen/TargetInstrInfo.h"
61 #include "llvm/CodeGen/TargetLowering.h"
62 #include "llvm/CodeGen/TargetOpcodes.h"
63 #include "llvm/CodeGen/TargetRegisterInfo.h"
64 #include "llvm/CodeGen/TargetSubtargetInfo.h"
65 #include "llvm/CodeGen/ValueTypes.h"
66 #include "llvm/IR/Attributes.h"
67 #include "llvm/IR/CallingConv.h"
68 #include "llvm/IR/Constant.h"
69 #include "llvm/IR/Constants.h"
70 #include "llvm/IR/DataLayout.h"
71 #include "llvm/IR/DebugLoc.h"
72 #include "llvm/IR/DerivedTypes.h"
73 #include "llvm/IR/Function.h"
74 #include "llvm/IR/GlobalAlias.h"
75 #include "llvm/IR/GlobalValue.h"
76 #include "llvm/IR/GlobalVariable.h"
77 #include "llvm/IR/IRBuilder.h"
78 #include "llvm/IR/InlineAsm.h"
79 #include "llvm/IR/Instruction.h"
80 #include "llvm/IR/Instructions.h"
81 #include "llvm/IR/IntrinsicInst.h"
82 #include "llvm/IR/Intrinsics.h"
83 #include "llvm/IR/IntrinsicsARM.h"
84 #include "llvm/IR/Module.h"
85 #include "llvm/IR/PatternMatch.h"
86 #include "llvm/IR/Type.h"
87 #include "llvm/IR/User.h"
88 #include "llvm/IR/Value.h"
89 #include "llvm/MC/MCInstrDesc.h"
90 #include "llvm/MC/MCInstrItineraries.h"
91 #include "llvm/MC/MCRegisterInfo.h"
92 #include "llvm/MC/MCSchedule.h"
93 #include "llvm/Support/AtomicOrdering.h"
94 #include "llvm/Support/BranchProbability.h"
95 #include "llvm/Support/Casting.h"
96 #include "llvm/Support/CodeGen.h"
97 #include "llvm/Support/CommandLine.h"
98 #include "llvm/Support/Compiler.h"
99 #include "llvm/Support/Debug.h"
100 #include "llvm/Support/ErrorHandling.h"
101 #include "llvm/Support/KnownBits.h"
102 #include "llvm/Support/MachineValueType.h"
103 #include "llvm/Support/MathExtras.h"
104 #include "llvm/Support/raw_ostream.h"
105 #include "llvm/Target/TargetMachine.h"
106 #include "llvm/Target/TargetOptions.h"
107 #include <algorithm>
108 #include <cassert>
109 #include <cstdint>
110 #include <cstdlib>
111 #include <iterator>
112 #include <limits>
113 #include <optional>
114 #include <string>
115 #include <tuple>
116 #include <utility>
117 #include <vector>
118
119 using namespace llvm;
120 using namespace llvm::PatternMatch;
121
122 #define DEBUG_TYPE "arm-isel"
123
124 STATISTIC(NumTailCalls, "Number of tail calls");
125 STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
126 STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
127 STATISTIC(NumConstpoolPromoted,
128 "Number of constants with their storage promoted into constant pools");
129
130 static cl::opt<bool>
131 ARMInterworking("arm-interworking", cl::Hidden,
132 cl::desc("Enable / disable ARM interworking (for debugging only)"),
133 cl::init(true));
134
135 static cl::opt<bool> EnableConstpoolPromotion(
136 "arm-promote-constant", cl::Hidden,
137 cl::desc("Enable / disable promotion of unnamed_addr constants into "
138 "constant pools"),
139 cl::init(false)); // FIXME: set to true by default once PR32780 is fixed
140 static cl::opt<unsigned> ConstpoolPromotionMaxSize(
141 "arm-promote-constant-max-size", cl::Hidden,
142 cl::desc("Maximum size of constant to promote into a constant pool"),
143 cl::init(64));
144 static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
145 "arm-promote-constant-max-total", cl::Hidden,
146 cl::desc("Maximum size of ALL constants to promote into a constant pool"),
147 cl::init(128));
148
149 cl::opt<unsigned>
150 MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
151 cl::desc("Maximum interleave factor for MVE VLDn to generate."),
152 cl::init(2));
153
154 // The APCS parameter registers.
155 static const MCPhysReg GPRArgRegs[] = {
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
157 };
158
addTypeForNEON(MVT VT,MVT PromotedLdStVT)159 void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) {
160 if (VT != PromotedLdStVT) {
161 setOperationAction(ISD::LOAD, VT, Promote);
162 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT);
163
164 setOperationAction(ISD::STORE, VT, Promote);
165 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT);
166 }
167
168 MVT ElemTy = VT.getVectorElementType();
169 if (ElemTy != MVT::f64)
170 setOperationAction(ISD::SETCC, VT, Custom);
171 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
172 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
173 if (ElemTy == MVT::i32) {
174 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
175 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
176 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
177 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
178 } else {
179 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
180 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
181 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
182 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
183 }
184 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
185 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
186 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
187 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
188 setOperationAction(ISD::SELECT, VT, Expand);
189 setOperationAction(ISD::SELECT_CC, VT, Expand);
190 setOperationAction(ISD::VSELECT, VT, Expand);
191 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
192 if (VT.isInteger()) {
193 setOperationAction(ISD::SHL, VT, Custom);
194 setOperationAction(ISD::SRA, VT, Custom);
195 setOperationAction(ISD::SRL, VT, Custom);
196 }
197
198 // Neon does not support vector divide/remainder operations.
199 setOperationAction(ISD::SDIV, VT, Expand);
200 setOperationAction(ISD::UDIV, VT, Expand);
201 setOperationAction(ISD::FDIV, VT, Expand);
202 setOperationAction(ISD::SREM, VT, Expand);
203 setOperationAction(ISD::UREM, VT, Expand);
204 setOperationAction(ISD::FREM, VT, Expand);
205 setOperationAction(ISD::SDIVREM, VT, Expand);
206 setOperationAction(ISD::UDIVREM, VT, Expand);
207
208 if (!VT.isFloatingPoint() &&
209 VT != MVT::v2i64 && VT != MVT::v1i64)
210 for (auto Opcode : {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
211 setOperationAction(Opcode, VT, Legal);
212 if (!VT.isFloatingPoint())
213 for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
214 setOperationAction(Opcode, VT, Legal);
215 }
216
addDRTypeForNEON(MVT VT)217 void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
218 addRegisterClass(VT, &ARM::DPRRegClass);
219 addTypeForNEON(VT, MVT::f64);
220 }
221
addQRTypeForNEON(MVT VT)222 void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
223 addRegisterClass(VT, &ARM::DPairRegClass);
224 addTypeForNEON(VT, MVT::v2f64);
225 }
226
setAllExpand(MVT VT)227 void ARMTargetLowering::setAllExpand(MVT VT) {
228 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
229 setOperationAction(Opc, VT, Expand);
230
231 // We support these really simple operations even on types where all
232 // the actual arithmetic has to be broken down into simpler
233 // operations or turned into library calls.
234 setOperationAction(ISD::BITCAST, VT, Legal);
235 setOperationAction(ISD::LOAD, VT, Legal);
236 setOperationAction(ISD::STORE, VT, Legal);
237 setOperationAction(ISD::UNDEF, VT, Legal);
238 }
239
addAllExtLoads(const MVT From,const MVT To,LegalizeAction Action)240 void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
241 LegalizeAction Action) {
242 setLoadExtAction(ISD::EXTLOAD, From, To, Action);
243 setLoadExtAction(ISD::ZEXTLOAD, From, To, Action);
244 setLoadExtAction(ISD::SEXTLOAD, From, To, Action);
245 }
246
addMVEVectorTypes(bool HasMVEFP)247 void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
248 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
249
250 for (auto VT : IntTypes) {
251 addRegisterClass(VT, &ARM::MQPRRegClass);
252 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
253 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
254 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
255 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
256 setOperationAction(ISD::SHL, VT, Custom);
257 setOperationAction(ISD::SRA, VT, Custom);
258 setOperationAction(ISD::SRL, VT, Custom);
259 setOperationAction(ISD::SMIN, VT, Legal);
260 setOperationAction(ISD::SMAX, VT, Legal);
261 setOperationAction(ISD::UMIN, VT, Legal);
262 setOperationAction(ISD::UMAX, VT, Legal);
263 setOperationAction(ISD::ABS, VT, Legal);
264 setOperationAction(ISD::SETCC, VT, Custom);
265 setOperationAction(ISD::MLOAD, VT, Custom);
266 setOperationAction(ISD::MSTORE, VT, Legal);
267 setOperationAction(ISD::CTLZ, VT, Legal);
268 setOperationAction(ISD::CTTZ, VT, Custom);
269 setOperationAction(ISD::BITREVERSE, VT, Legal);
270 setOperationAction(ISD::BSWAP, VT, Legal);
271 setOperationAction(ISD::SADDSAT, VT, Legal);
272 setOperationAction(ISD::UADDSAT, VT, Legal);
273 setOperationAction(ISD::SSUBSAT, VT, Legal);
274 setOperationAction(ISD::USUBSAT, VT, Legal);
275 setOperationAction(ISD::ABDS, VT, Legal);
276 setOperationAction(ISD::ABDU, VT, Legal);
277 setOperationAction(ISD::AVGFLOORS, VT, Legal);
278 setOperationAction(ISD::AVGFLOORU, VT, Legal);
279 setOperationAction(ISD::AVGCEILS, VT, Legal);
280 setOperationAction(ISD::AVGCEILU, VT, Legal);
281
282 // No native support for these.
283 setOperationAction(ISD::UDIV, VT, Expand);
284 setOperationAction(ISD::SDIV, VT, Expand);
285 setOperationAction(ISD::UREM, VT, Expand);
286 setOperationAction(ISD::SREM, VT, Expand);
287 setOperationAction(ISD::UDIVREM, VT, Expand);
288 setOperationAction(ISD::SDIVREM, VT, Expand);
289 setOperationAction(ISD::CTPOP, VT, Expand);
290 setOperationAction(ISD::SELECT, VT, Expand);
291 setOperationAction(ISD::SELECT_CC, VT, Expand);
292
293 // Vector reductions
294 setOperationAction(ISD::VECREDUCE_ADD, VT, Legal);
295 setOperationAction(ISD::VECREDUCE_SMAX, VT, Legal);
296 setOperationAction(ISD::VECREDUCE_UMAX, VT, Legal);
297 setOperationAction(ISD::VECREDUCE_SMIN, VT, Legal);
298 setOperationAction(ISD::VECREDUCE_UMIN, VT, Legal);
299 setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
300 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
301 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
302 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
303
304 if (!HasMVEFP) {
305 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
306 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
307 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
308 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
309 } else {
310 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
311 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
312 }
313
314 // Pre and Post inc are supported on loads and stores
315 for (unsigned im = (unsigned)ISD::PRE_INC;
316 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
317 setIndexedLoadAction(im, VT, Legal);
318 setIndexedStoreAction(im, VT, Legal);
319 setIndexedMaskedLoadAction(im, VT, Legal);
320 setIndexedMaskedStoreAction(im, VT, Legal);
321 }
322 }
323
324 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
325 for (auto VT : FloatTypes) {
326 addRegisterClass(VT, &ARM::MQPRRegClass);
327 if (!HasMVEFP)
328 setAllExpand(VT);
329
330 // These are legal or custom whether we have MVE.fp or not
331 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
332 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
333 setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getVectorElementType(), Custom);
334 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
335 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
336 setOperationAction(ISD::BUILD_VECTOR, VT.getVectorElementType(), Custom);
337 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal);
338 setOperationAction(ISD::SETCC, VT, Custom);
339 setOperationAction(ISD::MLOAD, VT, Custom);
340 setOperationAction(ISD::MSTORE, VT, Legal);
341 setOperationAction(ISD::SELECT, VT, Expand);
342 setOperationAction(ISD::SELECT_CC, VT, Expand);
343
344 // Pre and Post inc are supported on loads and stores
345 for (unsigned im = (unsigned)ISD::PRE_INC;
346 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
347 setIndexedLoadAction(im, VT, Legal);
348 setIndexedStoreAction(im, VT, Legal);
349 setIndexedMaskedLoadAction(im, VT, Legal);
350 setIndexedMaskedStoreAction(im, VT, Legal);
351 }
352
353 if (HasMVEFP) {
354 setOperationAction(ISD::FMINNUM, VT, Legal);
355 setOperationAction(ISD::FMAXNUM, VT, Legal);
356 setOperationAction(ISD::FROUND, VT, Legal);
357 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
358 setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom);
359 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
360 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
361
362 // No native support for these.
363 setOperationAction(ISD::FDIV, VT, Expand);
364 setOperationAction(ISD::FREM, VT, Expand);
365 setOperationAction(ISD::FSQRT, VT, Expand);
366 setOperationAction(ISD::FSIN, VT, Expand);
367 setOperationAction(ISD::FCOS, VT, Expand);
368 setOperationAction(ISD::FPOW, VT, Expand);
369 setOperationAction(ISD::FLOG, VT, Expand);
370 setOperationAction(ISD::FLOG2, VT, Expand);
371 setOperationAction(ISD::FLOG10, VT, Expand);
372 setOperationAction(ISD::FEXP, VT, Expand);
373 setOperationAction(ISD::FEXP2, VT, Expand);
374 setOperationAction(ISD::FNEARBYINT, VT, Expand);
375 }
376 }
377
378 // Custom Expand smaller than legal vector reductions to prevent false zero
379 // items being added.
380 setOperationAction(ISD::VECREDUCE_FADD, MVT::v4f16, Custom);
381 setOperationAction(ISD::VECREDUCE_FMUL, MVT::v4f16, Custom);
382 setOperationAction(ISD::VECREDUCE_FMIN, MVT::v4f16, Custom);
383 setOperationAction(ISD::VECREDUCE_FMAX, MVT::v4f16, Custom);
384 setOperationAction(ISD::VECREDUCE_FADD, MVT::v2f16, Custom);
385 setOperationAction(ISD::VECREDUCE_FMUL, MVT::v2f16, Custom);
386 setOperationAction(ISD::VECREDUCE_FMIN, MVT::v2f16, Custom);
387 setOperationAction(ISD::VECREDUCE_FMAX, MVT::v2f16, Custom);
388
389 // We 'support' these types up to bitcast/load/store level, regardless of
390 // MVE integer-only / float support. Only doing FP data processing on the FP
391 // vector types is inhibited at integer-only level.
392 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
393 for (auto VT : LongTypes) {
394 addRegisterClass(VT, &ARM::MQPRRegClass);
395 setAllExpand(VT);
396 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
397 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
398 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
399 setOperationAction(ISD::VSELECT, VT, Legal);
400 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
401 }
402 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
403
404 // We can do bitwise operations on v2i64 vectors
405 setOperationAction(ISD::AND, MVT::v2i64, Legal);
406 setOperationAction(ISD::OR, MVT::v2i64, Legal);
407 setOperationAction(ISD::XOR, MVT::v2i64, Legal);
408
409 // It is legal to extload from v4i8 to v4i16 or v4i32.
410 addAllExtLoads(MVT::v8i16, MVT::v8i8, Legal);
411 addAllExtLoads(MVT::v4i32, MVT::v4i16, Legal);
412 addAllExtLoads(MVT::v4i32, MVT::v4i8, Legal);
413
414 // It is legal to sign extend from v4i8/v4i16 to v4i32 or v8i8 to v8i16.
415 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Legal);
416 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal);
417 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal);
418 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i8, Legal);
419 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i16, Legal);
420
421 // Some truncating stores are legal too.
422 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
423 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
424 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
425
426 // Pre and Post inc on these are legal, given the correct extends
427 for (unsigned im = (unsigned)ISD::PRE_INC;
428 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
429 for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
430 setIndexedLoadAction(im, VT, Legal);
431 setIndexedStoreAction(im, VT, Legal);
432 setIndexedMaskedLoadAction(im, VT, Legal);
433 setIndexedMaskedStoreAction(im, VT, Legal);
434 }
435 }
436
437 // Predicate types
438 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
439 for (auto VT : pTypes) {
440 addRegisterClass(VT, &ARM::VCCRRegClass);
441 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
442 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
443 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
444 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
445 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
446 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
447 setOperationAction(ISD::SETCC, VT, Custom);
448 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
449 setOperationAction(ISD::LOAD, VT, Custom);
450 setOperationAction(ISD::STORE, VT, Custom);
451 setOperationAction(ISD::TRUNCATE, VT, Custom);
452 setOperationAction(ISD::VSELECT, VT, Expand);
453 setOperationAction(ISD::SELECT, VT, Expand);
454 setOperationAction(ISD::SELECT_CC, VT, Expand);
455
456 if (!HasMVEFP) {
457 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
458 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
459 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
460 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
461 }
462 }
463 setOperationAction(ISD::SETCC, MVT::v2i1, Expand);
464 setOperationAction(ISD::TRUNCATE, MVT::v2i1, Expand);
465 setOperationAction(ISD::AND, MVT::v2i1, Expand);
466 setOperationAction(ISD::OR, MVT::v2i1, Expand);
467 setOperationAction(ISD::XOR, MVT::v2i1, Expand);
468 setOperationAction(ISD::SINT_TO_FP, MVT::v2i1, Expand);
469 setOperationAction(ISD::UINT_TO_FP, MVT::v2i1, Expand);
470 setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Expand);
471 setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Expand);
472
473 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom);
474 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
475 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
476 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom);
477 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom);
478 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
479 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
480 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
481 }
482
ARMTargetLowering(const TargetMachine & TM,const ARMSubtarget & STI)483 ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
484 const ARMSubtarget &STI)
485 : TargetLowering(TM), Subtarget(&STI) {
486 RegInfo = Subtarget->getRegisterInfo();
487 Itins = Subtarget->getInstrItineraryData();
488
489 setBooleanContents(ZeroOrOneBooleanContent);
490 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
491
492 if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() &&
493 !Subtarget->isTargetWatchOS() && !Subtarget->isTargetDriverKit()) {
494 bool IsHFTarget = TM.Options.FloatABIType == FloatABI::Hard;
495 for (int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
496 setLibcallCallingConv(static_cast<RTLIB::Libcall>(LCID),
497 IsHFTarget ? CallingConv::ARM_AAPCS_VFP
498 : CallingConv::ARM_AAPCS);
499 }
500
501 if (Subtarget->isTargetMachO()) {
502 // Uses VFP for Thumb libfuncs if available.
503 if (Subtarget->isThumb() && Subtarget->hasVFP2Base() &&
504 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
505 static const struct {
506 const RTLIB::Libcall Op;
507 const char * const Name;
508 const ISD::CondCode Cond;
509 } LibraryCalls[] = {
510 // Single-precision floating-point arithmetic.
511 { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID },
512 { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID },
513 { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID },
514 { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID },
515
516 // Double-precision floating-point arithmetic.
517 { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID },
518 { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID },
519 { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID },
520 { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID },
521
522 // Single-precision comparisons.
523 { RTLIB::OEQ_F32, "__eqsf2vfp", ISD::SETNE },
524 { RTLIB::UNE_F32, "__nesf2vfp", ISD::SETNE },
525 { RTLIB::OLT_F32, "__ltsf2vfp", ISD::SETNE },
526 { RTLIB::OLE_F32, "__lesf2vfp", ISD::SETNE },
527 { RTLIB::OGE_F32, "__gesf2vfp", ISD::SETNE },
528 { RTLIB::OGT_F32, "__gtsf2vfp", ISD::SETNE },
529 { RTLIB::UO_F32, "__unordsf2vfp", ISD::SETNE },
530
531 // Double-precision comparisons.
532 { RTLIB::OEQ_F64, "__eqdf2vfp", ISD::SETNE },
533 { RTLIB::UNE_F64, "__nedf2vfp", ISD::SETNE },
534 { RTLIB::OLT_F64, "__ltdf2vfp", ISD::SETNE },
535 { RTLIB::OLE_F64, "__ledf2vfp", ISD::SETNE },
536 { RTLIB::OGE_F64, "__gedf2vfp", ISD::SETNE },
537 { RTLIB::OGT_F64, "__gtdf2vfp", ISD::SETNE },
538 { RTLIB::UO_F64, "__unorddf2vfp", ISD::SETNE },
539
540 // Floating-point to integer conversions.
541 // i64 conversions are done via library routines even when generating VFP
542 // instructions, so use the same ones.
543 { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp", ISD::SETCC_INVALID },
544 { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID },
545 { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp", ISD::SETCC_INVALID },
546 { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID },
547
548 // Conversions between floating types.
549 { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp", ISD::SETCC_INVALID },
550 { RTLIB::FPEXT_F32_F64, "__extendsfdf2vfp", ISD::SETCC_INVALID },
551
552 // Integer to floating-point conversions.
553 // i64 conversions are done via library routines even when generating VFP
554 // instructions, so use the same ones.
555 // FIXME: There appears to be some naming inconsistency in ARM libgcc:
556 // e.g., __floatunsidf vs. __floatunssidfvfp.
557 { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp", ISD::SETCC_INVALID },
558 { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID },
559 { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp", ISD::SETCC_INVALID },
560 { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID },
561 };
562
563 for (const auto &LC : LibraryCalls) {
564 setLibcallName(LC.Op, LC.Name);
565 if (LC.Cond != ISD::SETCC_INVALID)
566 setCmpLibcallCC(LC.Op, LC.Cond);
567 }
568 }
569 }
570
571 // These libcalls are not available in 32-bit.
572 setLibcallName(RTLIB::SHL_I128, nullptr);
573 setLibcallName(RTLIB::SRL_I128, nullptr);
574 setLibcallName(RTLIB::SRA_I128, nullptr);
575 setLibcallName(RTLIB::MUL_I128, nullptr);
576 setLibcallName(RTLIB::MULO_I64, nullptr);
577 setLibcallName(RTLIB::MULO_I128, nullptr);
578
579 // RTLIB
580 if (Subtarget->isAAPCS_ABI() &&
581 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() ||
582 Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) {
583 static const struct {
584 const RTLIB::Libcall Op;
585 const char * const Name;
586 const CallingConv::ID CC;
587 const ISD::CondCode Cond;
588 } LibraryCalls[] = {
589 // Double-precision floating-point arithmetic helper functions
590 // RTABI chapter 4.1.2, Table 2
591 { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
592 { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
593 { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
594 { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
595
596 // Double-precision floating-point comparison helper functions
597 // RTABI chapter 4.1.2, Table 3
598 { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
599 { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
600 { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
601 { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
602 { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
603 { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
604 { RTLIB::UO_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
605
606 // Single-precision floating-point arithmetic helper functions
607 // RTABI chapter 4.1.2, Table 4
608 { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
609 { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
610 { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
611 { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
612
613 // Single-precision floating-point comparison helper functions
614 // RTABI chapter 4.1.2, Table 5
615 { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
616 { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
617 { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
618 { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
619 { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
620 { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
621 { RTLIB::UO_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
622
623 // Floating-point to integer conversions.
624 // RTABI chapter 4.1.2, Table 6
625 { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
626 { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
627 { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
628 { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
629 { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
630 { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
631 { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
632 { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
633
634 // Conversions between floating types.
635 // RTABI chapter 4.1.2, Table 7
636 { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
637 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
638 { RTLIB::FPEXT_F32_F64, "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
639
640 // Integer to floating-point conversions.
641 // RTABI chapter 4.1.2, Table 8
642 { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
643 { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
644 { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
645 { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
646 { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
647 { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
648 { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
649 { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
650
651 // Long long helper functions
652 // RTABI chapter 4.2, Table 9
653 { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
654 { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
655 { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
656 { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
657
658 // Integer division functions
659 // RTABI chapter 4.3.1
660 { RTLIB::SDIV_I8, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
661 { RTLIB::SDIV_I16, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
662 { RTLIB::SDIV_I32, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
663 { RTLIB::SDIV_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
664 { RTLIB::UDIV_I8, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
665 { RTLIB::UDIV_I16, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
666 { RTLIB::UDIV_I32, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
667 { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
668 };
669
670 for (const auto &LC : LibraryCalls) {
671 setLibcallName(LC.Op, LC.Name);
672 setLibcallCallingConv(LC.Op, LC.CC);
673 if (LC.Cond != ISD::SETCC_INVALID)
674 setCmpLibcallCC(LC.Op, LC.Cond);
675 }
676
677 // EABI dependent RTLIB
678 if (TM.Options.EABIVersion == EABI::EABI4 ||
679 TM.Options.EABIVersion == EABI::EABI5) {
680 static const struct {
681 const RTLIB::Libcall Op;
682 const char *const Name;
683 const CallingConv::ID CC;
684 const ISD::CondCode Cond;
685 } MemOpsLibraryCalls[] = {
686 // Memory operations
687 // RTABI chapter 4.3.4
688 { RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
689 { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
690 { RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
691 };
692
693 for (const auto &LC : MemOpsLibraryCalls) {
694 setLibcallName(LC.Op, LC.Name);
695 setLibcallCallingConv(LC.Op, LC.CC);
696 if (LC.Cond != ISD::SETCC_INVALID)
697 setCmpLibcallCC(LC.Op, LC.Cond);
698 }
699 }
700 }
701
702 if (Subtarget->isTargetWindows()) {
703 static const struct {
704 const RTLIB::Libcall Op;
705 const char * const Name;
706 const CallingConv::ID CC;
707 } LibraryCalls[] = {
708 { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP },
709 { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP },
710 { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP },
711 { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP },
712 { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP },
713 { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP },
714 { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP },
715 { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP },
716 };
717
718 for (const auto &LC : LibraryCalls) {
719 setLibcallName(LC.Op, LC.Name);
720 setLibcallCallingConv(LC.Op, LC.CC);
721 }
722 }
723
724 // Use divmod compiler-rt calls for iOS 5.0 and later.
725 if (Subtarget->isTargetMachO() &&
726 !(Subtarget->isTargetIOS() &&
727 Subtarget->getTargetTriple().isOSVersionLT(5, 0))) {
728 setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
729 setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
730 }
731
732 // The half <-> float conversion functions are always soft-float on
733 // non-watchos platforms, but are needed for some targets which use a
734 // hard-float calling convention by default.
735 if (!Subtarget->isTargetWatchABI()) {
736 if (Subtarget->isAAPCS_ABI()) {
737 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
738 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
739 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS);
740 } else {
741 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS);
742 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS);
743 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS);
744 }
745 }
746
747 // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
748 // a __gnu_ prefix (which is the default).
749 if (Subtarget->isTargetAEABI()) {
750 static const struct {
751 const RTLIB::Libcall Op;
752 const char * const Name;
753 const CallingConv::ID CC;
754 } LibraryCalls[] = {
755 { RTLIB::FPROUND_F32_F16, "__aeabi_f2h", CallingConv::ARM_AAPCS },
756 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS },
757 { RTLIB::FPEXT_F16_F32, "__aeabi_h2f", CallingConv::ARM_AAPCS },
758 };
759
760 for (const auto &LC : LibraryCalls) {
761 setLibcallName(LC.Op, LC.Name);
762 setLibcallCallingConv(LC.Op, LC.CC);
763 }
764 }
765
766 if (Subtarget->isThumb1Only())
767 addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
768 else
769 addRegisterClass(MVT::i32, &ARM::GPRRegClass);
770
771 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
772 Subtarget->hasFPRegs()) {
773 addRegisterClass(MVT::f32, &ARM::SPRRegClass);
774 addRegisterClass(MVT::f64, &ARM::DPRRegClass);
775
776 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom);
777 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom);
778 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
779 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
780
781 if (!Subtarget->hasVFP2Base())
782 setAllExpand(MVT::f32);
783 if (!Subtarget->hasFP64())
784 setAllExpand(MVT::f64);
785 }
786
787 if (Subtarget->hasFullFP16()) {
788 addRegisterClass(MVT::f16, &ARM::HPRRegClass);
789 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
790 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
791
792 setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
793 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
794 }
795
796 if (Subtarget->hasBF16()) {
797 addRegisterClass(MVT::bf16, &ARM::HPRRegClass);
798 setAllExpand(MVT::bf16);
799 if (!Subtarget->hasFullFP16())
800 setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
801 }
802
803 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
804 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
805 setTruncStoreAction(VT, InnerVT, Expand);
806 addAllExtLoads(VT, InnerVT, Expand);
807 }
808
809 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
810 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
811
812 setOperationAction(ISD::BSWAP, VT, Expand);
813 }
814
815 setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
816 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
817
818 setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
819 setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);
820
821 if (Subtarget->hasMVEIntegerOps())
822 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
823
824 // Combine low-overhead loop intrinsics so that we can lower i1 types.
825 if (Subtarget->hasLOB()) {
826 setTargetDAGCombine({ISD::BRCOND, ISD::BR_CC});
827 }
828
829 if (Subtarget->hasNEON()) {
830 addDRTypeForNEON(MVT::v2f32);
831 addDRTypeForNEON(MVT::v8i8);
832 addDRTypeForNEON(MVT::v4i16);
833 addDRTypeForNEON(MVT::v2i32);
834 addDRTypeForNEON(MVT::v1i64);
835
836 addQRTypeForNEON(MVT::v4f32);
837 addQRTypeForNEON(MVT::v2f64);
838 addQRTypeForNEON(MVT::v16i8);
839 addQRTypeForNEON(MVT::v8i16);
840 addQRTypeForNEON(MVT::v4i32);
841 addQRTypeForNEON(MVT::v2i64);
842
843 if (Subtarget->hasFullFP16()) {
844 addQRTypeForNEON(MVT::v8f16);
845 addDRTypeForNEON(MVT::v4f16);
846 }
847
848 if (Subtarget->hasBF16()) {
849 addQRTypeForNEON(MVT::v8bf16);
850 addDRTypeForNEON(MVT::v4bf16);
851 }
852 }
853
854 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
855 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
856 // none of Neon, MVE or VFP supports any arithmetic operations on it.
857 setOperationAction(ISD::FADD, MVT::v2f64, Expand);
858 setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
859 setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
860 // FIXME: Code duplication: FDIV and FREM are expanded always, see
861 // ARMTargetLowering::addTypeForNEON method for details.
862 setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
863 setOperationAction(ISD::FREM, MVT::v2f64, Expand);
864 // FIXME: Create unittest.
865 // In another words, find a way when "copysign" appears in DAG with vector
866 // operands.
867 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand);
868 // FIXME: Code duplication: SETCC has custom operation action, see
869 // ARMTargetLowering::addTypeForNEON method for details.
870 setOperationAction(ISD::SETCC, MVT::v2f64, Expand);
871 // FIXME: Create unittest for FNEG and for FABS.
872 setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
873 setOperationAction(ISD::FABS, MVT::v2f64, Expand);
874 setOperationAction(ISD::FSQRT, MVT::v2f64, Expand);
875 setOperationAction(ISD::FSIN, MVT::v2f64, Expand);
876 setOperationAction(ISD::FCOS, MVT::v2f64, Expand);
877 setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
878 setOperationAction(ISD::FLOG, MVT::v2f64, Expand);
879 setOperationAction(ISD::FLOG2, MVT::v2f64, Expand);
880 setOperationAction(ISD::FLOG10, MVT::v2f64, Expand);
881 setOperationAction(ISD::FEXP, MVT::v2f64, Expand);
882 setOperationAction(ISD::FEXP2, MVT::v2f64, Expand);
883 // FIXME: Create unittest for FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR.
884 setOperationAction(ISD::FCEIL, MVT::v2f64, Expand);
885 setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand);
886 setOperationAction(ISD::FRINT, MVT::v2f64, Expand);
887 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand);
888 setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand);
889 setOperationAction(ISD::FMA, MVT::v2f64, Expand);
890 }
891
892 if (Subtarget->hasNEON()) {
893 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
894 // supported for v4f32.
895 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
896 setOperationAction(ISD::FSIN, MVT::v4f32, Expand);
897 setOperationAction(ISD::FCOS, MVT::v4f32, Expand);
898 setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
899 setOperationAction(ISD::FLOG, MVT::v4f32, Expand);
900 setOperationAction(ISD::FLOG2, MVT::v4f32, Expand);
901 setOperationAction(ISD::FLOG10, MVT::v4f32, Expand);
902 setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
903 setOperationAction(ISD::FEXP2, MVT::v4f32, Expand);
904 setOperationAction(ISD::FCEIL, MVT::v4f32, Expand);
905 setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand);
906 setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
907 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
908 setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand);
909
910 // Mark v2f32 intrinsics.
911 setOperationAction(ISD::FSQRT, MVT::v2f32, Expand);
912 setOperationAction(ISD::FSIN, MVT::v2f32, Expand);
913 setOperationAction(ISD::FCOS, MVT::v2f32, Expand);
914 setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
915 setOperationAction(ISD::FLOG, MVT::v2f32, Expand);
916 setOperationAction(ISD::FLOG2, MVT::v2f32, Expand);
917 setOperationAction(ISD::FLOG10, MVT::v2f32, Expand);
918 setOperationAction(ISD::FEXP, MVT::v2f32, Expand);
919 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand);
920 setOperationAction(ISD::FCEIL, MVT::v2f32, Expand);
921 setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand);
922 setOperationAction(ISD::FRINT, MVT::v2f32, Expand);
923 setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand);
924 setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand);
925
926 // Neon does not support some operations on v1i64 and v2i64 types.
927 setOperationAction(ISD::MUL, MVT::v1i64, Expand);
928 // Custom handling for some quad-vector types to detect VMULL.
929 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
930 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
931 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
932 // Custom handling for some vector types to avoid expensive expansions
933 setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
934 setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
935 setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
936 setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
937 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
938 // a destination type that is wider than the source, and nor does
939 // it have a FP_TO_[SU]INT instruction with a narrower destination than
940 // source.
941 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
942 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
943 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
944 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
945 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom);
946 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
947 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom);
948 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
949
950 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand);
951 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand);
952
953 // NEON does not have single instruction CTPOP for vectors with element
954 // types wider than 8-bits. However, custom lowering can leverage the
955 // v8i8/v16i8 vcnt instruction.
956 setOperationAction(ISD::CTPOP, MVT::v2i32, Custom);
957 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
958 setOperationAction(ISD::CTPOP, MVT::v4i16, Custom);
959 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom);
960 setOperationAction(ISD::CTPOP, MVT::v1i64, Custom);
961 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom);
962
963 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
964 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
965
966 // NEON does not have single instruction CTTZ for vectors.
967 setOperationAction(ISD::CTTZ, MVT::v8i8, Custom);
968 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
969 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
970 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
971
972 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
973 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
974 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
975 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
976
977 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom);
978 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom);
979 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom);
980 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom);
981
982 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom);
983 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom);
984 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom);
985 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom);
986
987 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
988 setOperationAction(ISD::MULHS, VT, Expand);
989 setOperationAction(ISD::MULHU, VT, Expand);
990 }
991
992 // NEON only has FMA instructions as of VFP4.
993 if (!Subtarget->hasVFP4Base()) {
994 setOperationAction(ISD::FMA, MVT::v2f32, Expand);
995 setOperationAction(ISD::FMA, MVT::v4f32, Expand);
996 }
997
998 setTargetDAGCombine({ISD::SHL, ISD::SRL, ISD::SRA, ISD::FP_TO_SINT,
999 ISD::FP_TO_UINT, ISD::FDIV, ISD::LOAD});
1000
1001 // It is legal to extload from v4i8 to v4i16 or v4i32.
1002 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1003 MVT::v2i32}) {
1004 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1005 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal);
1006 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal);
1007 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal);
1008 }
1009 }
1010 }
1011
1012 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1013 setTargetDAGCombine(
1014 {ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE, ISD::INSERT_SUBVECTOR,
1015 ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1016 ISD::SIGN_EXTEND_INREG, ISD::STORE, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,
1017 ISD::ANY_EXTEND, ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
1018 ISD::INTRINSIC_VOID, ISD::VECREDUCE_ADD, ISD::ADD, ISD::BITCAST});
1019 }
1020 if (Subtarget->hasMVEIntegerOps()) {
1021 setTargetDAGCombine({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
1022 ISD::FP_EXTEND, ISD::SELECT, ISD::SELECT_CC,
1023 ISD::SETCC});
1024 }
1025 if (Subtarget->hasMVEFloatOps()) {
1026 setTargetDAGCombine(ISD::FADD);
1027 }
1028
1029 if (!Subtarget->hasFP64()) {
1030 // When targeting a floating-point unit with only single-precision
1031 // operations, f64 is legal for the few double-precision instructions which
1032 // are present However, no double-precision operations other than moves,
1033 // loads and stores are provided by the hardware.
1034 setOperationAction(ISD::FADD, MVT::f64, Expand);
1035 setOperationAction(ISD::FSUB, MVT::f64, Expand);
1036 setOperationAction(ISD::FMUL, MVT::f64, Expand);
1037 setOperationAction(ISD::FMA, MVT::f64, Expand);
1038 setOperationAction(ISD::FDIV, MVT::f64, Expand);
1039 setOperationAction(ISD::FREM, MVT::f64, Expand);
1040 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
1041 setOperationAction(ISD::FGETSIGN, MVT::f64, Expand);
1042 setOperationAction(ISD::FNEG, MVT::f64, Expand);
1043 setOperationAction(ISD::FABS, MVT::f64, Expand);
1044 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
1045 setOperationAction(ISD::FSIN, MVT::f64, Expand);
1046 setOperationAction(ISD::FCOS, MVT::f64, Expand);
1047 setOperationAction(ISD::FPOW, MVT::f64, Expand);
1048 setOperationAction(ISD::FLOG, MVT::f64, Expand);
1049 setOperationAction(ISD::FLOG2, MVT::f64, Expand);
1050 setOperationAction(ISD::FLOG10, MVT::f64, Expand);
1051 setOperationAction(ISD::FEXP, MVT::f64, Expand);
1052 setOperationAction(ISD::FEXP2, MVT::f64, Expand);
1053 setOperationAction(ISD::FCEIL, MVT::f64, Expand);
1054 setOperationAction(ISD::FTRUNC, MVT::f64, Expand);
1055 setOperationAction(ISD::FRINT, MVT::f64, Expand);
1056 setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand);
1057 setOperationAction(ISD::FFLOOR, MVT::f64, Expand);
1058 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
1059 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
1060 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
1061 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
1062 setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom);
1063 setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom);
1064 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
1065 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
1066 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
1067 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::f64, Custom);
1068 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::f64, Custom);
1069 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
1070 }
1071
1072 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
1073 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
1074 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
1075 if (Subtarget->hasFullFP16()) {
1076 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
1077 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
1078 }
1079 }
1080
1081 if (!Subtarget->hasFP16()) {
1082 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
1083 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
1084 }
1085
1086 computeRegisterProperties(Subtarget->getRegisterInfo());
1087
1088 // ARM does not have floating-point extending loads.
1089 for (MVT VT : MVT::fp_valuetypes()) {
1090 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1091 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
1092 }
1093
1094 // ... or truncating stores
1095 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
1096 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
1097 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
1098
1099 // ARM does not have i1 sign extending load.
1100 for (MVT VT : MVT::integer_valuetypes())
1101 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
1102
1103 // ARM supports all 4 flavors of integer indexed load / store.
1104 if (!Subtarget->isThumb1Only()) {
1105 for (unsigned im = (unsigned)ISD::PRE_INC;
1106 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
1107 setIndexedLoadAction(im, MVT::i1, Legal);
1108 setIndexedLoadAction(im, MVT::i8, Legal);
1109 setIndexedLoadAction(im, MVT::i16, Legal);
1110 setIndexedLoadAction(im, MVT::i32, Legal);
1111 setIndexedStoreAction(im, MVT::i1, Legal);
1112 setIndexedStoreAction(im, MVT::i8, Legal);
1113 setIndexedStoreAction(im, MVT::i16, Legal);
1114 setIndexedStoreAction(im, MVT::i32, Legal);
1115 }
1116 } else {
1117 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
1118 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
1119 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
1120 }
1121
1122 setOperationAction(ISD::SADDO, MVT::i32, Custom);
1123 setOperationAction(ISD::UADDO, MVT::i32, Custom);
1124 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
1125 setOperationAction(ISD::USUBO, MVT::i32, Custom);
1126
1127 setOperationAction(ISD::ADDCARRY, MVT::i32, Custom);
1128 setOperationAction(ISD::SUBCARRY, MVT::i32, Custom);
1129 if (Subtarget->hasDSP()) {
1130 setOperationAction(ISD::SADDSAT, MVT::i8, Custom);
1131 setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
1132 setOperationAction(ISD::SADDSAT, MVT::i16, Custom);
1133 setOperationAction(ISD::SSUBSAT, MVT::i16, Custom);
1134 setOperationAction(ISD::UADDSAT, MVT::i8, Custom);
1135 setOperationAction(ISD::USUBSAT, MVT::i8, Custom);
1136 setOperationAction(ISD::UADDSAT, MVT::i16, Custom);
1137 setOperationAction(ISD::USUBSAT, MVT::i16, Custom);
1138 }
1139 if (Subtarget->hasBaseDSP()) {
1140 setOperationAction(ISD::SADDSAT, MVT::i32, Legal);
1141 setOperationAction(ISD::SSUBSAT, MVT::i32, Legal);
1142 }
1143
1144 // i64 operation support.
1145 setOperationAction(ISD::MUL, MVT::i64, Expand);
1146 setOperationAction(ISD::MULHU, MVT::i32, Expand);
1147 if (Subtarget->isThumb1Only()) {
1148 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
1149 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
1150 }
1151 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
1152 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
1153 setOperationAction(ISD::MULHS, MVT::i32, Expand);
1154
1155 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
1156 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
1157 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
1158 setOperationAction(ISD::SRL, MVT::i64, Custom);
1159 setOperationAction(ISD::SRA, MVT::i64, Custom);
1160 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1161 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
1162 setOperationAction(ISD::LOAD, MVT::i64, Custom);
1163 setOperationAction(ISD::STORE, MVT::i64, Custom);
1164
1165 // MVE lowers 64 bit shifts to lsll and lsrl
1166 // assuming that ISD::SRL and SRA of i64 are already marked custom
1167 if (Subtarget->hasMVEIntegerOps())
1168 setOperationAction(ISD::SHL, MVT::i64, Custom);
1169
1170 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
1171 if (Subtarget->isThumb1Only()) {
1172 setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
1173 setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
1174 setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
1175 }
1176
1177 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
1178 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
1179
1180 // ARM does not have ROTL.
1181 setOperationAction(ISD::ROTL, MVT::i32, Expand);
1182 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1183 setOperationAction(ISD::ROTL, VT, Expand);
1184 setOperationAction(ISD::ROTR, VT, Expand);
1185 }
1186 setOperationAction(ISD::CTTZ, MVT::i32, Custom);
1187 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
1188 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
1189 setOperationAction(ISD::CTLZ, MVT::i32, Expand);
1190 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, LibCall);
1191 }
1192
1193 // @llvm.readcyclecounter requires the Performance Monitors extension.
1194 // Default to the 0 expansion on unsupported platforms.
1195 // FIXME: Technically there are older ARM CPUs that have
1196 // implementation-specific ways of obtaining this information.
1197 if (Subtarget->hasPerfMon())
1198 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
1199
1200 // Only ARMv6 has BSWAP.
1201 if (!Subtarget->hasV6Ops())
1202 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
1203
1204 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1205 : Subtarget->hasDivideInARMMode();
1206 if (!hasDivide) {
1207 // These are expanded into libcalls if the cpu doesn't have HW divider.
1208 setOperationAction(ISD::SDIV, MVT::i32, LibCall);
1209 setOperationAction(ISD::UDIV, MVT::i32, LibCall);
1210 }
1211
1212 if (Subtarget->isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1213 setOperationAction(ISD::SDIV, MVT::i32, Custom);
1214 setOperationAction(ISD::UDIV, MVT::i32, Custom);
1215
1216 setOperationAction(ISD::SDIV, MVT::i64, Custom);
1217 setOperationAction(ISD::UDIV, MVT::i64, Custom);
1218 }
1219
1220 setOperationAction(ISD::SREM, MVT::i32, Expand);
1221 setOperationAction(ISD::UREM, MVT::i32, Expand);
1222
1223 // Register based DivRem for AEABI (RTABI 4.2)
1224 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
1225 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
1226 Subtarget->isTargetWindows()) {
1227 setOperationAction(ISD::SREM, MVT::i64, Custom);
1228 setOperationAction(ISD::UREM, MVT::i64, Custom);
1229 HasStandaloneRem = false;
1230
1231 if (Subtarget->isTargetWindows()) {
1232 const struct {
1233 const RTLIB::Libcall Op;
1234 const char * const Name;
1235 const CallingConv::ID CC;
1236 } LibraryCalls[] = {
1237 { RTLIB::SDIVREM_I8, "__rt_sdiv", CallingConv::ARM_AAPCS },
1238 { RTLIB::SDIVREM_I16, "__rt_sdiv", CallingConv::ARM_AAPCS },
1239 { RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS },
1240 { RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS },
1241
1242 { RTLIB::UDIVREM_I8, "__rt_udiv", CallingConv::ARM_AAPCS },
1243 { RTLIB::UDIVREM_I16, "__rt_udiv", CallingConv::ARM_AAPCS },
1244 { RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS },
1245 { RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS },
1246 };
1247
1248 for (const auto &LC : LibraryCalls) {
1249 setLibcallName(LC.Op, LC.Name);
1250 setLibcallCallingConv(LC.Op, LC.CC);
1251 }
1252 } else {
1253 const struct {
1254 const RTLIB::Libcall Op;
1255 const char * const Name;
1256 const CallingConv::ID CC;
1257 } LibraryCalls[] = {
1258 { RTLIB::SDIVREM_I8, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
1259 { RTLIB::SDIVREM_I16, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
1260 { RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
1261 { RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS },
1262
1263 { RTLIB::UDIVREM_I8, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
1264 { RTLIB::UDIVREM_I16, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
1265 { RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
1266 { RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS },
1267 };
1268
1269 for (const auto &LC : LibraryCalls) {
1270 setLibcallName(LC.Op, LC.Name);
1271 setLibcallCallingConv(LC.Op, LC.CC);
1272 }
1273 }
1274
1275 setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
1276 setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
1277 setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
1278 setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
1279 } else {
1280 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
1281 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
1282 }
1283
1284 if (Subtarget->getTargetTriple().isOSMSVCRT()) {
1285 // MSVCRT doesn't have powi; fall back to pow
1286 setLibcallName(RTLIB::POWI_F32, nullptr);
1287 setLibcallName(RTLIB::POWI_F64, nullptr);
1288 }
1289
1290 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
1291 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
1292 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
1293 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
1294
1295 setOperationAction(ISD::TRAP, MVT::Other, Legal);
1296 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
1297
1298 // Use the default implementation.
1299 setOperationAction(ISD::VASTART, MVT::Other, Custom);
1300 setOperationAction(ISD::VAARG, MVT::Other, Expand);
1301 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
1302 setOperationAction(ISD::VAEND, MVT::Other, Expand);
1303 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
1304 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
1305
1306 if (Subtarget->isTargetWindows())
1307 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
1308 else
1309 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
1310
1311 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
1312 // the default expansion.
1313 InsertFencesForAtomic = false;
1314 if (Subtarget->hasAnyDataBarrier() &&
1315 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1316 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
1317 // to ldrex/strex loops already.
1318 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
1319 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1320 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom);
1321
1322 // On v8, we have particularly efficient implementations of atomic fences
1323 // if they can be combined with nearby atomic loads and stores.
1324 if (!Subtarget->hasAcquireRelease() ||
1325 getTargetMachine().getOptLevel() == 0) {
1326 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
1327 InsertFencesForAtomic = true;
1328 }
1329 } else {
1330 // If there's anything we can use as a barrier, go through custom lowering
1331 // for ATOMIC_FENCE.
1332 // If target has DMB in thumb, Fences can be inserted.
1333 if (Subtarget->hasDataBarrier())
1334 InsertFencesForAtomic = true;
1335
1336 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other,
1337 Subtarget->hasAnyDataBarrier() ? Custom : Expand);
1338
1339 // Set them all for expansion, which will force libcalls.
1340 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Expand);
1341 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Expand);
1342 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Expand);
1343 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
1344 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Expand);
1345 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Expand);
1346 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Expand);
1347 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand);
1348 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand);
1349 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand);
1350 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand);
1351 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand);
1352 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
1353 // Unordered/Monotonic case.
1354 if (!InsertFencesForAtomic) {
1355 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
1356 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
1357 }
1358 }
1359
1360 // Compute supported atomic widths.
1361 if (Subtarget->isTargetLinux() ||
1362 (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
1363 // For targets where __sync_* routines are reliably available, we use them
1364 // if necessary.
1365 //
1366 // ARM Linux always supports 64-bit atomics through kernel-assisted atomic
1367 // routines (kernel 3.1 or later). FIXME: Not with compiler-rt?
1368 //
1369 // ARMv6 targets have native instructions in ARM mode. For Thumb mode,
1370 // such targets should provide __sync_* routines, which use the ARM mode
1371 // instructions. (ARMv6 doesn't have dmb, but it has an equivalent
1372 // encoding; see ARMISD::MEMBARRIER_MCR.)
1373 setMaxAtomicSizeInBitsSupported(64);
1374 } else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) ||
1375 Subtarget->hasForced32BitAtomics()) {
1376 // Cortex-M (besides Cortex-M0) have 32-bit atomics.
1377 setMaxAtomicSizeInBitsSupported(32);
1378 } else {
1379 // We can't assume anything about other targets; just use libatomic
1380 // routines.
1381 setMaxAtomicSizeInBitsSupported(0);
1382 }
1383
1384 setMaxDivRemBitWidthSupported(64);
1385
1386 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
1387
1388 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
1389 if (!Subtarget->hasV6Ops()) {
1390 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
1391 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
1392 }
1393 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
1394
1395 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1396 !Subtarget->isThumb1Only()) {
1397 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
1398 // iff target supports vfp2.
1399 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1400 setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
1401 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
1402 }
1403
1404 // We want to custom lower some of our intrinsics.
1405 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
1406 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
1407 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
1408 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
1409 if (Subtarget->useSjLjEH())
1410 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
1411
1412 setOperationAction(ISD::SETCC, MVT::i32, Expand);
1413 setOperationAction(ISD::SETCC, MVT::f32, Expand);
1414 setOperationAction(ISD::SETCC, MVT::f64, Expand);
1415 setOperationAction(ISD::SELECT, MVT::i32, Custom);
1416 setOperationAction(ISD::SELECT, MVT::f32, Custom);
1417 setOperationAction(ISD::SELECT, MVT::f64, Custom);
1418 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
1419 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
1420 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
1421 if (Subtarget->hasFullFP16()) {
1422 setOperationAction(ISD::SETCC, MVT::f16, Expand);
1423 setOperationAction(ISD::SELECT, MVT::f16, Custom);
1424 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
1425 }
1426
1427 setOperationAction(ISD::SETCCCARRY, MVT::i32, Custom);
1428
1429 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
1430 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
1431 if (Subtarget->hasFullFP16())
1432 setOperationAction(ISD::BR_CC, MVT::f16, Custom);
1433 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
1434 setOperationAction(ISD::BR_CC, MVT::f64, Custom);
1435 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
1436
1437 // We don't support sin/cos/fmod/copysign/pow
1438 setOperationAction(ISD::FSIN, MVT::f64, Expand);
1439 setOperationAction(ISD::FSIN, MVT::f32, Expand);
1440 setOperationAction(ISD::FCOS, MVT::f32, Expand);
1441 setOperationAction(ISD::FCOS, MVT::f64, Expand);
1442 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
1443 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
1444 setOperationAction(ISD::FREM, MVT::f64, Expand);
1445 setOperationAction(ISD::FREM, MVT::f32, Expand);
1446 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
1447 !Subtarget->isThumb1Only()) {
1448 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
1449 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
1450 }
1451 setOperationAction(ISD::FPOW, MVT::f64, Expand);
1452 setOperationAction(ISD::FPOW, MVT::f32, Expand);
1453
1454 if (!Subtarget->hasVFP4Base()) {
1455 setOperationAction(ISD::FMA, MVT::f64, Expand);
1456 setOperationAction(ISD::FMA, MVT::f32, Expand);
1457 }
1458
1459 // Various VFP goodness
1460 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
1461 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
1462 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
1463 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
1464 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
1465 }
1466
1467 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
1468 if (!Subtarget->hasFP16()) {
1469 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
1470 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
1471 }
1472
1473 // Strict floating-point comparisons need custom lowering.
1474 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
1475 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
1476 setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
1477 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
1478 setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
1479 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
1480 }
1481
1482 // Use __sincos_stret if available.
1483 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
1484 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
1485 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1486 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1487 }
1488
1489 // FP-ARMv8 implements a lot of rounding-like FP operations.
1490 if (Subtarget->hasFPARMv8Base()) {
1491 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
1492 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
1493 setOperationAction(ISD::FROUND, MVT::f32, Legal);
1494 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
1495 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
1496 setOperationAction(ISD::FRINT, MVT::f32, Legal);
1497 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
1498 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
1499 if (Subtarget->hasNEON()) {
1500 setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal);
1501 setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal);
1502 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
1503 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
1504 }
1505
1506 if (Subtarget->hasFP64()) {
1507 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
1508 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
1509 setOperationAction(ISD::FROUND, MVT::f64, Legal);
1510 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
1511 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
1512 setOperationAction(ISD::FRINT, MVT::f64, Legal);
1513 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
1514 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
1515 }
1516 }
1517
1518 // FP16 often need to be promoted to call lib functions
1519 if (Subtarget->hasFullFP16()) {
1520 setOperationAction(ISD::FREM, MVT::f16, Promote);
1521 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
1522 setOperationAction(ISD::FSIN, MVT::f16, Promote);
1523 setOperationAction(ISD::FCOS, MVT::f16, Promote);
1524 setOperationAction(ISD::FSINCOS, MVT::f16, Promote);
1525 setOperationAction(ISD::FPOWI, MVT::f16, Promote);
1526 setOperationAction(ISD::FPOW, MVT::f16, Promote);
1527 setOperationAction(ISD::FEXP, MVT::f16, Promote);
1528 setOperationAction(ISD::FEXP2, MVT::f16, Promote);
1529 setOperationAction(ISD::FLOG, MVT::f16, Promote);
1530 setOperationAction(ISD::FLOG10, MVT::f16, Promote);
1531 setOperationAction(ISD::FLOG2, MVT::f16, Promote);
1532
1533 setOperationAction(ISD::FROUND, MVT::f16, Legal);
1534 }
1535
1536 if (Subtarget->hasNEON()) {
1537 // vmin and vmax aren't available in a scalar form, so we can use
1538 // a NEON instruction with an undef lane instead. This has a performance
1539 // penalty on some cores, so we don't do this unless we have been
1540 // asked to by the core tuning model.
1541 if (Subtarget->useNEONForSinglePrecisionFP()) {
1542 setOperationAction(ISD::FMINIMUM, MVT::f32, Legal);
1543 setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal);
1544 setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
1545 setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
1546 }
1547 setOperationAction(ISD::FMINIMUM, MVT::v2f32, Legal);
1548 setOperationAction(ISD::FMAXIMUM, MVT::v2f32, Legal);
1549 setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
1550 setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
1551
1552 if (Subtarget->hasFullFP16()) {
1553 setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal);
1554 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal);
1555 setOperationAction(ISD::FMINNUM, MVT::v8f16, Legal);
1556 setOperationAction(ISD::FMAXNUM, MVT::v8f16, Legal);
1557
1558 setOperationAction(ISD::FMINIMUM, MVT::v4f16, Legal);
1559 setOperationAction(ISD::FMAXIMUM, MVT::v4f16, Legal);
1560 setOperationAction(ISD::FMINIMUM, MVT::v8f16, Legal);
1561 setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Legal);
1562 }
1563 }
1564
1565 // We have target-specific dag combine patterns for the following nodes:
1566 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
1567 setTargetDAGCombine(
1568 {ISD::ADD, ISD::SUB, ISD::MUL, ISD::AND, ISD::OR, ISD::XOR});
1569
1570 if (Subtarget->hasMVEIntegerOps())
1571 setTargetDAGCombine(ISD::VSELECT);
1572
1573 if (Subtarget->hasV6Ops())
1574 setTargetDAGCombine(ISD::SRL);
1575 if (Subtarget->isThumb1Only())
1576 setTargetDAGCombine(ISD::SHL);
1577 // Attempt to lower smin/smax to ssat/usat
1578 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1579 Subtarget->isThumb2()) {
1580 setTargetDAGCombine({ISD::SMIN, ISD::SMAX});
1581 }
1582
1583 setStackPointerRegisterToSaveRestore(ARM::SP);
1584
1585 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1586 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
1587 setSchedulingPreference(Sched::RegPressure);
1588 else
1589 setSchedulingPreference(Sched::Hybrid);
1590
1591 //// temporary - rewrite interface to use type
1592 MaxStoresPerMemset = 8;
1593 MaxStoresPerMemsetOptSize = 4;
1594 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
1595 MaxStoresPerMemcpyOptSize = 2;
1596 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
1597 MaxStoresPerMemmoveOptSize = 2;
1598
1599 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1600 // are at least 4 bytes aligned.
1601 setMinStackArgumentAlignment(Align(4));
1602
1603 // Prefer likely predicted branches to selects on out-of-order cores.
1604 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
1605
1606 setPrefLoopAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment()));
1607
1608 setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
1609
1610 if (Subtarget->isThumb() || Subtarget->isThumb2())
1611 setTargetDAGCombine(ISD::ABS);
1612 }
1613
useSoftFloat() const1614 bool ARMTargetLowering::useSoftFloat() const {
1615 return Subtarget->useSoftFloat();
1616 }
1617
1618 // FIXME: It might make sense to define the representative register class as the
1619 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1620 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1621 // SPR's representative would be DPR_VFP2. This should work well if register
1622 // pressure tracking were modified such that a register use would increment the
1623 // pressure of the register class's representative and all of it's super
1624 // classes' representatives transitively. We have not implemented this because
1625 // of the difficulty prior to coalescing of modeling operand register classes
1626 // due to the common occurrence of cross class copies and subregister insertions
1627 // and extractions.
1628 std::pair<const TargetRegisterClass *, uint8_t>
findRepresentativeClass(const TargetRegisterInfo * TRI,MVT VT) const1629 ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
1630 MVT VT) const {
1631 const TargetRegisterClass *RRC = nullptr;
1632 uint8_t Cost = 1;
1633 switch (VT.SimpleTy) {
1634 default:
1635 return TargetLowering::findRepresentativeClass(TRI, VT);
1636 // Use DPR as representative register class for all floating point
1637 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1638 // the cost is 1 for both f32 and f64.
1639 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
1640 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
1641 RRC = &ARM::DPRRegClass;
1642 // When NEON is used for SP, only half of the register file is available
1643 // because operations that define both SP and DP results will be constrained
1644 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1645 // coalescing by double-counting the SP regs. See the FIXME above.
1646 if (Subtarget->useNEONForSinglePrecisionFP())
1647 Cost = 2;
1648 break;
1649 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1650 case MVT::v4f32: case MVT::v2f64:
1651 RRC = &ARM::DPRRegClass;
1652 Cost = 2;
1653 break;
1654 case MVT::v4i64:
1655 RRC = &ARM::DPRRegClass;
1656 Cost = 4;
1657 break;
1658 case MVT::v8i64:
1659 RRC = &ARM::DPRRegClass;
1660 Cost = 8;
1661 break;
1662 }
1663 return std::make_pair(RRC, Cost);
1664 }
1665
getTargetNodeName(unsigned Opcode) const1666 const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
1667 #define MAKE_CASE(V) \
1668 case V: \
1669 return #V;
1670 switch ((ARMISD::NodeType)Opcode) {
1671 case ARMISD::FIRST_NUMBER:
1672 break;
1673 MAKE_CASE(ARMISD::Wrapper)
1674 MAKE_CASE(ARMISD::WrapperPIC)
1675 MAKE_CASE(ARMISD::WrapperJT)
1676 MAKE_CASE(ARMISD::COPY_STRUCT_BYVAL)
1677 MAKE_CASE(ARMISD::CALL)
1678 MAKE_CASE(ARMISD::CALL_PRED)
1679 MAKE_CASE(ARMISD::CALL_NOLINK)
1680 MAKE_CASE(ARMISD::tSECALL)
1681 MAKE_CASE(ARMISD::t2CALL_BTI)
1682 MAKE_CASE(ARMISD::BRCOND)
1683 MAKE_CASE(ARMISD::BR_JT)
1684 MAKE_CASE(ARMISD::BR2_JT)
1685 MAKE_CASE(ARMISD::RET_FLAG)
1686 MAKE_CASE(ARMISD::SERET_FLAG)
1687 MAKE_CASE(ARMISD::INTRET_FLAG)
1688 MAKE_CASE(ARMISD::PIC_ADD)
1689 MAKE_CASE(ARMISD::CMP)
1690 MAKE_CASE(ARMISD::CMN)
1691 MAKE_CASE(ARMISD::CMPZ)
1692 MAKE_CASE(ARMISD::CMPFP)
1693 MAKE_CASE(ARMISD::CMPFPE)
1694 MAKE_CASE(ARMISD::CMPFPw0)
1695 MAKE_CASE(ARMISD::CMPFPEw0)
1696 MAKE_CASE(ARMISD::BCC_i64)
1697 MAKE_CASE(ARMISD::FMSTAT)
1698 MAKE_CASE(ARMISD::CMOV)
1699 MAKE_CASE(ARMISD::SUBS)
1700 MAKE_CASE(ARMISD::SSAT)
1701 MAKE_CASE(ARMISD::USAT)
1702 MAKE_CASE(ARMISD::ASRL)
1703 MAKE_CASE(ARMISD::LSRL)
1704 MAKE_CASE(ARMISD::LSLL)
1705 MAKE_CASE(ARMISD::SRL_FLAG)
1706 MAKE_CASE(ARMISD::SRA_FLAG)
1707 MAKE_CASE(ARMISD::RRX)
1708 MAKE_CASE(ARMISD::ADDC)
1709 MAKE_CASE(ARMISD::ADDE)
1710 MAKE_CASE(ARMISD::SUBC)
1711 MAKE_CASE(ARMISD::SUBE)
1712 MAKE_CASE(ARMISD::LSLS)
1713 MAKE_CASE(ARMISD::VMOVRRD)
1714 MAKE_CASE(ARMISD::VMOVDRR)
1715 MAKE_CASE(ARMISD::VMOVhr)
1716 MAKE_CASE(ARMISD::VMOVrh)
1717 MAKE_CASE(ARMISD::VMOVSR)
1718 MAKE_CASE(ARMISD::EH_SJLJ_SETJMP)
1719 MAKE_CASE(ARMISD::EH_SJLJ_LONGJMP)
1720 MAKE_CASE(ARMISD::EH_SJLJ_SETUP_DISPATCH)
1721 MAKE_CASE(ARMISD::TC_RETURN)
1722 MAKE_CASE(ARMISD::THREAD_POINTER)
1723 MAKE_CASE(ARMISD::DYN_ALLOC)
1724 MAKE_CASE(ARMISD::MEMBARRIER_MCR)
1725 MAKE_CASE(ARMISD::PRELOAD)
1726 MAKE_CASE(ARMISD::LDRD)
1727 MAKE_CASE(ARMISD::STRD)
1728 MAKE_CASE(ARMISD::WIN__CHKSTK)
1729 MAKE_CASE(ARMISD::WIN__DBZCHK)
1730 MAKE_CASE(ARMISD::PREDICATE_CAST)
1731 MAKE_CASE(ARMISD::VECTOR_REG_CAST)
1732 MAKE_CASE(ARMISD::MVESEXT)
1733 MAKE_CASE(ARMISD::MVEZEXT)
1734 MAKE_CASE(ARMISD::MVETRUNC)
1735 MAKE_CASE(ARMISD::VCMP)
1736 MAKE_CASE(ARMISD::VCMPZ)
1737 MAKE_CASE(ARMISD::VTST)
1738 MAKE_CASE(ARMISD::VSHLs)
1739 MAKE_CASE(ARMISD::VSHLu)
1740 MAKE_CASE(ARMISD::VSHLIMM)
1741 MAKE_CASE(ARMISD::VSHRsIMM)
1742 MAKE_CASE(ARMISD::VSHRuIMM)
1743 MAKE_CASE(ARMISD::VRSHRsIMM)
1744 MAKE_CASE(ARMISD::VRSHRuIMM)
1745 MAKE_CASE(ARMISD::VRSHRNIMM)
1746 MAKE_CASE(ARMISD::VQSHLsIMM)
1747 MAKE_CASE(ARMISD::VQSHLuIMM)
1748 MAKE_CASE(ARMISD::VQSHLsuIMM)
1749 MAKE_CASE(ARMISD::VQSHRNsIMM)
1750 MAKE_CASE(ARMISD::VQSHRNuIMM)
1751 MAKE_CASE(ARMISD::VQSHRNsuIMM)
1752 MAKE_CASE(ARMISD::VQRSHRNsIMM)
1753 MAKE_CASE(ARMISD::VQRSHRNuIMM)
1754 MAKE_CASE(ARMISD::VQRSHRNsuIMM)
1755 MAKE_CASE(ARMISD::VSLIIMM)
1756 MAKE_CASE(ARMISD::VSRIIMM)
1757 MAKE_CASE(ARMISD::VGETLANEu)
1758 MAKE_CASE(ARMISD::VGETLANEs)
1759 MAKE_CASE(ARMISD::VMOVIMM)
1760 MAKE_CASE(ARMISD::VMVNIMM)
1761 MAKE_CASE(ARMISD::VMOVFPIMM)
1762 MAKE_CASE(ARMISD::VDUP)
1763 MAKE_CASE(ARMISD::VDUPLANE)
1764 MAKE_CASE(ARMISD::VEXT)
1765 MAKE_CASE(ARMISD::VREV64)
1766 MAKE_CASE(ARMISD::VREV32)
1767 MAKE_CASE(ARMISD::VREV16)
1768 MAKE_CASE(ARMISD::VZIP)
1769 MAKE_CASE(ARMISD::VUZP)
1770 MAKE_CASE(ARMISD::VTRN)
1771 MAKE_CASE(ARMISD::VTBL1)
1772 MAKE_CASE(ARMISD::VTBL2)
1773 MAKE_CASE(ARMISD::VMOVN)
1774 MAKE_CASE(ARMISD::VQMOVNs)
1775 MAKE_CASE(ARMISD::VQMOVNu)
1776 MAKE_CASE(ARMISD::VCVTN)
1777 MAKE_CASE(ARMISD::VCVTL)
1778 MAKE_CASE(ARMISD::VIDUP)
1779 MAKE_CASE(ARMISD::VMULLs)
1780 MAKE_CASE(ARMISD::VMULLu)
1781 MAKE_CASE(ARMISD::VQDMULH)
1782 MAKE_CASE(ARMISD::VADDVs)
1783 MAKE_CASE(ARMISD::VADDVu)
1784 MAKE_CASE(ARMISD::VADDVps)
1785 MAKE_CASE(ARMISD::VADDVpu)
1786 MAKE_CASE(ARMISD::VADDLVs)
1787 MAKE_CASE(ARMISD::VADDLVu)
1788 MAKE_CASE(ARMISD::VADDLVAs)
1789 MAKE_CASE(ARMISD::VADDLVAu)
1790 MAKE_CASE(ARMISD::VADDLVps)
1791 MAKE_CASE(ARMISD::VADDLVpu)
1792 MAKE_CASE(ARMISD::VADDLVAps)
1793 MAKE_CASE(ARMISD::VADDLVApu)
1794 MAKE_CASE(ARMISD::VMLAVs)
1795 MAKE_CASE(ARMISD::VMLAVu)
1796 MAKE_CASE(ARMISD::VMLAVps)
1797 MAKE_CASE(ARMISD::VMLAVpu)
1798 MAKE_CASE(ARMISD::VMLALVs)
1799 MAKE_CASE(ARMISD::VMLALVu)
1800 MAKE_CASE(ARMISD::VMLALVps)
1801 MAKE_CASE(ARMISD::VMLALVpu)
1802 MAKE_CASE(ARMISD::VMLALVAs)
1803 MAKE_CASE(ARMISD::VMLALVAu)
1804 MAKE_CASE(ARMISD::VMLALVAps)
1805 MAKE_CASE(ARMISD::VMLALVApu)
1806 MAKE_CASE(ARMISD::VMINVu)
1807 MAKE_CASE(ARMISD::VMINVs)
1808 MAKE_CASE(ARMISD::VMAXVu)
1809 MAKE_CASE(ARMISD::VMAXVs)
1810 MAKE_CASE(ARMISD::UMAAL)
1811 MAKE_CASE(ARMISD::UMLAL)
1812 MAKE_CASE(ARMISD::SMLAL)
1813 MAKE_CASE(ARMISD::SMLALBB)
1814 MAKE_CASE(ARMISD::SMLALBT)
1815 MAKE_CASE(ARMISD::SMLALTB)
1816 MAKE_CASE(ARMISD::SMLALTT)
1817 MAKE_CASE(ARMISD::SMULWB)
1818 MAKE_CASE(ARMISD::SMULWT)
1819 MAKE_CASE(ARMISD::SMLALD)
1820 MAKE_CASE(ARMISD::SMLALDX)
1821 MAKE_CASE(ARMISD::SMLSLD)
1822 MAKE_CASE(ARMISD::SMLSLDX)
1823 MAKE_CASE(ARMISD::SMMLAR)
1824 MAKE_CASE(ARMISD::SMMLSR)
1825 MAKE_CASE(ARMISD::QADD16b)
1826 MAKE_CASE(ARMISD::QSUB16b)
1827 MAKE_CASE(ARMISD::QADD8b)
1828 MAKE_CASE(ARMISD::QSUB8b)
1829 MAKE_CASE(ARMISD::UQADD16b)
1830 MAKE_CASE(ARMISD::UQSUB16b)
1831 MAKE_CASE(ARMISD::UQADD8b)
1832 MAKE_CASE(ARMISD::UQSUB8b)
1833 MAKE_CASE(ARMISD::BUILD_VECTOR)
1834 MAKE_CASE(ARMISD::BFI)
1835 MAKE_CASE(ARMISD::VORRIMM)
1836 MAKE_CASE(ARMISD::VBICIMM)
1837 MAKE_CASE(ARMISD::VBSP)
1838 MAKE_CASE(ARMISD::MEMCPY)
1839 MAKE_CASE(ARMISD::VLD1DUP)
1840 MAKE_CASE(ARMISD::VLD2DUP)
1841 MAKE_CASE(ARMISD::VLD3DUP)
1842 MAKE_CASE(ARMISD::VLD4DUP)
1843 MAKE_CASE(ARMISD::VLD1_UPD)
1844 MAKE_CASE(ARMISD::VLD2_UPD)
1845 MAKE_CASE(ARMISD::VLD3_UPD)
1846 MAKE_CASE(ARMISD::VLD4_UPD)
1847 MAKE_CASE(ARMISD::VLD1x2_UPD)
1848 MAKE_CASE(ARMISD::VLD1x3_UPD)
1849 MAKE_CASE(ARMISD::VLD1x4_UPD)
1850 MAKE_CASE(ARMISD::VLD2LN_UPD)
1851 MAKE_CASE(ARMISD::VLD3LN_UPD)
1852 MAKE_CASE(ARMISD::VLD4LN_UPD)
1853 MAKE_CASE(ARMISD::VLD1DUP_UPD)
1854 MAKE_CASE(ARMISD::VLD2DUP_UPD)
1855 MAKE_CASE(ARMISD::VLD3DUP_UPD)
1856 MAKE_CASE(ARMISD::VLD4DUP_UPD)
1857 MAKE_CASE(ARMISD::VST1_UPD)
1858 MAKE_CASE(ARMISD::VST2_UPD)
1859 MAKE_CASE(ARMISD::VST3_UPD)
1860 MAKE_CASE(ARMISD::VST4_UPD)
1861 MAKE_CASE(ARMISD::VST1x2_UPD)
1862 MAKE_CASE(ARMISD::VST1x3_UPD)
1863 MAKE_CASE(ARMISD::VST1x4_UPD)
1864 MAKE_CASE(ARMISD::VST2LN_UPD)
1865 MAKE_CASE(ARMISD::VST3LN_UPD)
1866 MAKE_CASE(ARMISD::VST4LN_UPD)
1867 MAKE_CASE(ARMISD::WLS)
1868 MAKE_CASE(ARMISD::WLSSETUP)
1869 MAKE_CASE(ARMISD::LE)
1870 MAKE_CASE(ARMISD::LOOP_DEC)
1871 MAKE_CASE(ARMISD::CSINV)
1872 MAKE_CASE(ARMISD::CSNEG)
1873 MAKE_CASE(ARMISD::CSINC)
1874 MAKE_CASE(ARMISD::MEMCPYLOOP)
1875 MAKE_CASE(ARMISD::MEMSETLOOP)
1876 #undef MAKE_CASE
1877 }
1878 return nullptr;
1879 }
1880
getSetCCResultType(const DataLayout & DL,LLVMContext &,EVT VT) const1881 EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1882 EVT VT) const {
1883 if (!VT.isVector())
1884 return getPointerTy(DL);
1885
1886 // MVE has a predicate register.
1887 if ((Subtarget->hasMVEIntegerOps() &&
1888 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1889 VT == MVT::v16i8)) ||
1890 (Subtarget->hasMVEFloatOps() &&
1891 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1892 return MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1893 return VT.changeVectorElementTypeToInteger();
1894 }
1895
1896 /// getRegClassFor - Return the register class that should be used for the
1897 /// specified value type.
1898 const TargetRegisterClass *
getRegClassFor(MVT VT,bool isDivergent) const1899 ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
1900 (void)isDivergent;
1901 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1902 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1903 // load / store 4 to 8 consecutive NEON D registers, or 2 to 4 consecutive
1904 // MVE Q registers.
1905 if (Subtarget->hasNEON()) {
1906 if (VT == MVT::v4i64)
1907 return &ARM::QQPRRegClass;
1908 if (VT == MVT::v8i64)
1909 return &ARM::QQQQPRRegClass;
1910 }
1911 if (Subtarget->hasMVEIntegerOps()) {
1912 if (VT == MVT::v4i64)
1913 return &ARM::MQQPRRegClass;
1914 if (VT == MVT::v8i64)
1915 return &ARM::MQQQQPRRegClass;
1916 }
1917 return TargetLowering::getRegClassFor(VT);
1918 }
1919
1920 // memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1921 // source/dest is aligned and the copy size is large enough. We therefore want
1922 // to align such objects passed to memory intrinsics.
shouldAlignPointerArgs(CallInst * CI,unsigned & MinSize,Align & PrefAlign) const1923 bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
1924 Align &PrefAlign) const {
1925 if (!isa<MemIntrinsic>(CI))
1926 return false;
1927 MinSize = 8;
1928 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1929 // cycle faster than 4-byte aligned LDM.
1930 PrefAlign =
1931 (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? Align(8) : Align(4));
1932 return true;
1933 }
1934
1935 // Create a fast isel object.
1936 FastISel *
createFastISel(FunctionLoweringInfo & funcInfo,const TargetLibraryInfo * libInfo) const1937 ARMTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1938 const TargetLibraryInfo *libInfo) const {
1939 return ARM::createFastISel(funcInfo, libInfo);
1940 }
1941
getSchedulingPreference(SDNode * N) const1942 Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
1943 unsigned NumVals = N->getNumValues();
1944 if (!NumVals)
1945 return Sched::RegPressure;
1946
1947 for (unsigned i = 0; i != NumVals; ++i) {
1948 EVT VT = N->getValueType(i);
1949 if (VT == MVT::Glue || VT == MVT::Other)
1950 continue;
1951 if (VT.isFloatingPoint() || VT.isVector())
1952 return Sched::ILP;
1953 }
1954
1955 if (!N->isMachineOpcode())
1956 return Sched::RegPressure;
1957
1958 // Load are scheduled for latency even if there instruction itinerary
1959 // is not available.
1960 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1961 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
1962
1963 if (MCID.getNumDefs() == 0)
1964 return Sched::RegPressure;
1965 if (!Itins->isEmpty() &&
1966 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2)
1967 return Sched::ILP;
1968
1969 return Sched::RegPressure;
1970 }
1971
1972 //===----------------------------------------------------------------------===//
1973 // Lowering Code
1974 //===----------------------------------------------------------------------===//
1975
isSRL16(const SDValue & Op)1976 static bool isSRL16(const SDValue &Op) {
1977 if (Op.getOpcode() != ISD::SRL)
1978 return false;
1979 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1980 return Const->getZExtValue() == 16;
1981 return false;
1982 }
1983
isSRA16(const SDValue & Op)1984 static bool isSRA16(const SDValue &Op) {
1985 if (Op.getOpcode() != ISD::SRA)
1986 return false;
1987 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1988 return Const->getZExtValue() == 16;
1989 return false;
1990 }
1991
isSHL16(const SDValue & Op)1992 static bool isSHL16(const SDValue &Op) {
1993 if (Op.getOpcode() != ISD::SHL)
1994 return false;
1995 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1996 return Const->getZExtValue() == 16;
1997 return false;
1998 }
1999
2000 // Check for a signed 16-bit value. We special case SRA because it makes it
2001 // more simple when also looking for SRAs that aren't sign extending a
2002 // smaller value. Without the check, we'd need to take extra care with
2003 // checking order for some operations.
isS16(const SDValue & Op,SelectionDAG & DAG)2004 static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
2005 if (isSRA16(Op))
2006 return isSHL16(Op.getOperand(0));
2007 return DAG.ComputeNumSignBits(Op) == 17;
2008 }
2009
2010 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
IntCCToARMCC(ISD::CondCode CC)2011 static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
2012 switch (CC) {
2013 default: llvm_unreachable("Unknown condition code!");
2014 case ISD::SETNE: return ARMCC::NE;
2015 case ISD::SETEQ: return ARMCC::EQ;
2016 case ISD::SETGT: return ARMCC::GT;
2017 case ISD::SETGE: return ARMCC::GE;
2018 case ISD::SETLT: return ARMCC::LT;
2019 case ISD::SETLE: return ARMCC::LE;
2020 case ISD::SETUGT: return ARMCC::HI;
2021 case ISD::SETUGE: return ARMCC::HS;
2022 case ISD::SETULT: return ARMCC::LO;
2023 case ISD::SETULE: return ARMCC::LS;
2024 }
2025 }
2026
2027 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
FPCCToARMCC(ISD::CondCode CC,ARMCC::CondCodes & CondCode,ARMCC::CondCodes & CondCode2)2028 static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
2029 ARMCC::CondCodes &CondCode2) {
2030 CondCode2 = ARMCC::AL;
2031 switch (CC) {
2032 default: llvm_unreachable("Unknown FP condition!");
2033 case ISD::SETEQ:
2034 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
2035 case ISD::SETGT:
2036 case ISD::SETOGT: CondCode = ARMCC::GT; break;
2037 case ISD::SETGE:
2038 case ISD::SETOGE: CondCode = ARMCC::GE; break;
2039 case ISD::SETOLT: CondCode = ARMCC::MI; break;
2040 case ISD::SETOLE: CondCode = ARMCC::LS; break;
2041 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
2042 case ISD::SETO: CondCode = ARMCC::VC; break;
2043 case ISD::SETUO: CondCode = ARMCC::VS; break;
2044 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
2045 case ISD::SETUGT: CondCode = ARMCC::HI; break;
2046 case ISD::SETUGE: CondCode = ARMCC::PL; break;
2047 case ISD::SETLT:
2048 case ISD::SETULT: CondCode = ARMCC::LT; break;
2049 case ISD::SETLE:
2050 case ISD::SETULE: CondCode = ARMCC::LE; break;
2051 case ISD::SETNE:
2052 case ISD::SETUNE: CondCode = ARMCC::NE; break;
2053 }
2054 }
2055
2056 //===----------------------------------------------------------------------===//
2057 // Calling Convention Implementation
2058 //===----------------------------------------------------------------------===//
2059
2060 /// getEffectiveCallingConv - Get the effective calling convention, taking into
2061 /// account presence of floating point hardware and calling convention
2062 /// limitations, such as support for variadic functions.
2063 CallingConv::ID
getEffectiveCallingConv(CallingConv::ID CC,bool isVarArg) const2064 ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
2065 bool isVarArg) const {
2066 switch (CC) {
2067 default:
2068 report_fatal_error("Unsupported calling convention");
2069 case CallingConv::ARM_AAPCS:
2070 case CallingConv::ARM_APCS:
2071 case CallingConv::GHC:
2072 case CallingConv::CFGuard_Check:
2073 return CC;
2074 case CallingConv::PreserveMost:
2075 return CallingConv::PreserveMost;
2076 case CallingConv::ARM_AAPCS_VFP:
2077 case CallingConv::Swift:
2078 case CallingConv::SwiftTail:
2079 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
2080 case CallingConv::C:
2081 case CallingConv::Tail:
2082 if (!Subtarget->isAAPCS_ABI())
2083 return CallingConv::ARM_APCS;
2084 else if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() &&
2085 getTargetMachine().Options.FloatABIType == FloatABI::Hard &&
2086 !isVarArg)
2087 return CallingConv::ARM_AAPCS_VFP;
2088 else
2089 return CallingConv::ARM_AAPCS;
2090 case CallingConv::Fast:
2091 case CallingConv::CXX_FAST_TLS:
2092 if (!Subtarget->isAAPCS_ABI()) {
2093 if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() && !isVarArg)
2094 return CallingConv::Fast;
2095 return CallingConv::ARM_APCS;
2096 } else if (Subtarget->hasVFP2Base() &&
2097 !Subtarget->isThumb1Only() && !isVarArg)
2098 return CallingConv::ARM_AAPCS_VFP;
2099 else
2100 return CallingConv::ARM_AAPCS;
2101 }
2102 }
2103
CCAssignFnForCall(CallingConv::ID CC,bool isVarArg) const2104 CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
2105 bool isVarArg) const {
2106 return CCAssignFnForNode(CC, false, isVarArg);
2107 }
2108
CCAssignFnForReturn(CallingConv::ID CC,bool isVarArg) const2109 CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
2110 bool isVarArg) const {
2111 return CCAssignFnForNode(CC, true, isVarArg);
2112 }
2113
2114 /// CCAssignFnForNode - Selects the correct CCAssignFn for the given
2115 /// CallingConvention.
CCAssignFnForNode(CallingConv::ID CC,bool Return,bool isVarArg) const2116 CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
2117 bool Return,
2118 bool isVarArg) const {
2119 switch (getEffectiveCallingConv(CC, isVarArg)) {
2120 default:
2121 report_fatal_error("Unsupported calling convention");
2122 case CallingConv::ARM_APCS:
2123 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
2124 case CallingConv::ARM_AAPCS:
2125 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
2126 case CallingConv::ARM_AAPCS_VFP:
2127 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
2128 case CallingConv::Fast:
2129 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
2130 case CallingConv::GHC:
2131 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
2132 case CallingConv::PreserveMost:
2133 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
2134 case CallingConv::CFGuard_Check:
2135 return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
2136 }
2137 }
2138
MoveToHPR(const SDLoc & dl,SelectionDAG & DAG,MVT LocVT,MVT ValVT,SDValue Val) const2139 SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
2140 MVT LocVT, MVT ValVT, SDValue Val) const {
2141 Val = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocVT.getSizeInBits()),
2142 Val);
2143 if (Subtarget->hasFullFP16()) {
2144 Val = DAG.getNode(ARMISD::VMOVhr, dl, ValVT, Val);
2145 } else {
2146 Val = DAG.getNode(ISD::TRUNCATE, dl,
2147 MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
2148 Val = DAG.getNode(ISD::BITCAST, dl, ValVT, Val);
2149 }
2150 return Val;
2151 }
2152
MoveFromHPR(const SDLoc & dl,SelectionDAG & DAG,MVT LocVT,MVT ValVT,SDValue Val) const2153 SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
2154 MVT LocVT, MVT ValVT,
2155 SDValue Val) const {
2156 if (Subtarget->hasFullFP16()) {
2157 Val = DAG.getNode(ARMISD::VMOVrh, dl,
2158 MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
2159 } else {
2160 Val = DAG.getNode(ISD::BITCAST, dl,
2161 MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
2162 Val = DAG.getNode(ISD::ZERO_EXTEND, dl,
2163 MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
2164 }
2165 return DAG.getNode(ISD::BITCAST, dl, LocVT, Val);
2166 }
2167
2168 /// LowerCallResult - Lower the result values of a call into the
2169 /// appropriate copies out of appropriate physical registers.
LowerCallResult(SDValue Chain,SDValue InFlag,CallingConv::ID CallConv,bool isVarArg,const SmallVectorImpl<ISD::InputArg> & Ins,const SDLoc & dl,SelectionDAG & DAG,SmallVectorImpl<SDValue> & InVals,bool isThisReturn,SDValue ThisVal) const2170 SDValue ARMTargetLowering::LowerCallResult(
2171 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2172 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
2173 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
2174 SDValue ThisVal) const {
2175 // Assign locations to each value returned by this call.
2176 SmallVector<CCValAssign, 16> RVLocs;
2177 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2178 *DAG.getContext());
2179 CCInfo.AnalyzeCallResult(Ins, CCAssignFnForReturn(CallConv, isVarArg));
2180
2181 // Copy all of the result registers out of their specified physreg.
2182 for (unsigned i = 0; i != RVLocs.size(); ++i) {
2183 CCValAssign VA = RVLocs[i];
2184
2185 // Pass 'this' value directly from the argument to return value, to avoid
2186 // reg unit interference
2187 if (i == 0 && isThisReturn) {
2188 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
2189 "unexpected return calling convention register assignment");
2190 InVals.push_back(ThisVal);
2191 continue;
2192 }
2193
2194 SDValue Val;
2195 if (VA.needsCustom() &&
2196 (VA.getLocVT() == MVT::f64 || VA.getLocVT() == MVT::v2f64)) {
2197 // Handle f64 or half of a v2f64.
2198 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2199 InFlag);
2200 Chain = Lo.getValue(1);
2201 InFlag = Lo.getValue(2);
2202 VA = RVLocs[++i]; // skip ahead to next loc
2203 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2204 InFlag);
2205 Chain = Hi.getValue(1);
2206 InFlag = Hi.getValue(2);
2207 if (!Subtarget->isLittle())
2208 std::swap (Lo, Hi);
2209 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
2210
2211 if (VA.getLocVT() == MVT::v2f64) {
2212 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
2213 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
2214 DAG.getConstant(0, dl, MVT::i32));
2215
2216 VA = RVLocs[++i]; // skip ahead to next loc
2217 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
2218 Chain = Lo.getValue(1);
2219 InFlag = Lo.getValue(2);
2220 VA = RVLocs[++i]; // skip ahead to next loc
2221 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
2222 Chain = Hi.getValue(1);
2223 InFlag = Hi.getValue(2);
2224 if (!Subtarget->isLittle())
2225 std::swap (Lo, Hi);
2226 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
2227 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
2228 DAG.getConstant(1, dl, MVT::i32));
2229 }
2230 } else {
2231 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
2232 InFlag);
2233 Chain = Val.getValue(1);
2234 InFlag = Val.getValue(2);
2235 }
2236
2237 switch (VA.getLocInfo()) {
2238 default: llvm_unreachable("Unknown loc info!");
2239 case CCValAssign::Full: break;
2240 case CCValAssign::BCvt:
2241 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val);
2242 break;
2243 }
2244
2245 // f16 arguments have their size extended to 4 bytes and passed as if they
2246 // had been copied to the LSBs of a 32-bit register.
2247 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2248 if (VA.needsCustom() &&
2249 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
2250 Val = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Val);
2251
2252 InVals.push_back(Val);
2253 }
2254
2255 return Chain;
2256 }
2257
computeAddrForCallArg(const SDLoc & dl,SelectionDAG & DAG,const CCValAssign & VA,SDValue StackPtr,bool IsTailCall,int SPDiff) const2258 std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2259 const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
2260 bool IsTailCall, int SPDiff) const {
2261 SDValue DstAddr;
2262 MachinePointerInfo DstInfo;
2263 int32_t Offset = VA.getLocMemOffset();
2264 MachineFunction &MF = DAG.getMachineFunction();
2265
2266 if (IsTailCall) {
2267 Offset += SPDiff;
2268 auto PtrVT = getPointerTy(DAG.getDataLayout());
2269 int Size = VA.getLocVT().getFixedSizeInBits() / 8;
2270 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
2271 DstAddr = DAG.getFrameIndex(FI, PtrVT);
2272 DstInfo =
2273 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
2274 } else {
2275 SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
2276 DstAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
2277 StackPtr, PtrOff);
2278 DstInfo =
2279 MachinePointerInfo::getStack(DAG.getMachineFunction(), Offset);
2280 }
2281
2282 return std::make_pair(DstAddr, DstInfo);
2283 }
2284
PassF64ArgInRegs(const SDLoc & dl,SelectionDAG & DAG,SDValue Chain,SDValue & Arg,RegsToPassVector & RegsToPass,CCValAssign & VA,CCValAssign & NextVA,SDValue & StackPtr,SmallVectorImpl<SDValue> & MemOpChains,bool IsTailCall,int SPDiff) const2285 void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
2286 SDValue Chain, SDValue &Arg,
2287 RegsToPassVector &RegsToPass,
2288 CCValAssign &VA, CCValAssign &NextVA,
2289 SDValue &StackPtr,
2290 SmallVectorImpl<SDValue> &MemOpChains,
2291 bool IsTailCall,
2292 int SPDiff) const {
2293 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
2294 DAG.getVTList(MVT::i32, MVT::i32), Arg);
2295 unsigned id = Subtarget->isLittle() ? 0 : 1;
2296 RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id)));
2297
2298 if (NextVA.isRegLoc())
2299 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id)));
2300 else {
2301 assert(NextVA.isMemLoc());
2302 if (!StackPtr.getNode())
2303 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
2304 getPointerTy(DAG.getDataLayout()));
2305
2306 SDValue DstAddr;
2307 MachinePointerInfo DstInfo;
2308 std::tie(DstAddr, DstInfo) =
2309 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2310 MemOpChains.push_back(
2311 DAG.getStore(Chain, dl, fmrrd.getValue(1 - id), DstAddr, DstInfo));
2312 }
2313 }
2314
canGuaranteeTCO(CallingConv::ID CC,bool GuaranteeTailCalls)2315 static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
2316 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
2317 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
2318 }
2319
2320 /// LowerCall - Lowering a call into a callseq_start <-
2321 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
2322 /// nodes.
2323 SDValue
LowerCall(TargetLowering::CallLoweringInfo & CLI,SmallVectorImpl<SDValue> & InVals) const2324 ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2325 SmallVectorImpl<SDValue> &InVals) const {
2326 SelectionDAG &DAG = CLI.DAG;
2327 SDLoc &dl = CLI.DL;
2328 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2329 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2330 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2331 SDValue Chain = CLI.Chain;
2332 SDValue Callee = CLI.Callee;
2333 bool &isTailCall = CLI.IsTailCall;
2334 CallingConv::ID CallConv = CLI.CallConv;
2335 bool doesNotRet = CLI.DoesNotReturn;
2336 bool isVarArg = CLI.IsVarArg;
2337
2338 MachineFunction &MF = DAG.getMachineFunction();
2339 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2340 MachineFunction::CallSiteInfo CSInfo;
2341 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
2342 bool isThisReturn = false;
2343 bool isCmseNSCall = false;
2344 bool isSibCall = false;
2345 bool PreferIndirect = false;
2346 bool GuardWithBTI = false;
2347
2348 // Lower 'returns_twice' calls to a pseudo-instruction.
2349 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) &&
2350 !Subtarget->noBTIAtReturnTwice())
2351 GuardWithBTI = AFI->branchTargetEnforcement();
2352
2353 // Determine whether this is a non-secure function call.
2354 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr("cmse_nonsecure_call"))
2355 isCmseNSCall = true;
2356
2357 // Disable tail calls if they're not supported.
2358 if (!Subtarget->supportsTailCall())
2359 isTailCall = false;
2360
2361 // For both the non-secure calls and the returns from a CMSE entry function,
2362 // the function needs to do some extra work afte r the call, or before the
2363 // return, respectively, thus it cannot end with atail call
2364 if (isCmseNSCall || AFI->isCmseNSEntryFunction())
2365 isTailCall = false;
2366
2367 if (isa<GlobalAddressSDNode>(Callee)) {
2368 // If we're optimizing for minimum size and the function is called three or
2369 // more times in this block, we can improve codesize by calling indirectly
2370 // as BLXr has a 16-bit encoding.
2371 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2372 if (CLI.CB) {
2373 auto *BB = CLI.CB->getParent();
2374 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
2375 count_if(GV->users(), [&BB](const User *U) {
2376 return isa<Instruction>(U) &&
2377 cast<Instruction>(U)->getParent() == BB;
2378 }) > 2;
2379 }
2380 }
2381 if (isTailCall) {
2382 // Check if it's really possible to do a tail call.
2383 isTailCall = IsEligibleForTailCallOptimization(
2384 Callee, CallConv, isVarArg, isStructRet,
2385 MF.getFunction().hasStructRetAttr(), Outs, OutVals, Ins, DAG,
2386 PreferIndirect);
2387
2388 if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
2389 CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
2390 isSibCall = true;
2391
2392 // We don't support GuaranteedTailCallOpt for ARM, only automatically
2393 // detected sibcalls.
2394 if (isTailCall)
2395 ++NumTailCalls;
2396 }
2397
2398 if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
2399 report_fatal_error("failed to perform tail call elimination on a call "
2400 "site marked musttail");
2401 // Analyze operands of the call, assigning locations to each operand.
2402 SmallVector<CCValAssign, 16> ArgLocs;
2403 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2404 *DAG.getContext());
2405 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CallConv, isVarArg));
2406
2407 // Get a count of how many bytes are to be pushed on the stack.
2408 unsigned NumBytes = CCInfo.getNextStackOffset();
2409
2410 // SPDiff is the byte offset of the call's argument area from the callee's.
2411 // Stores to callee stack arguments will be placed in FixedStackSlots offset
2412 // by this amount for a tail call. In a sibling call it must be 0 because the
2413 // caller will deallocate the entire stack and the callee still expects its
2414 // arguments to begin at SP+0. Completely unused for non-tail calls.
2415 int SPDiff = 0;
2416
2417 if (isTailCall && !isSibCall) {
2418 auto FuncInfo = MF.getInfo<ARMFunctionInfo>();
2419 unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
2420
2421 // Since callee will pop argument stack as a tail call, we must keep the
2422 // popped size 16-byte aligned.
2423 Align StackAlign = DAG.getDataLayout().getStackAlignment();
2424 NumBytes = alignTo(NumBytes, StackAlign);
2425
2426 // SPDiff will be negative if this tail call requires more space than we
2427 // would automatically have in our incoming argument space. Positive if we
2428 // can actually shrink the stack.
2429 SPDiff = NumReusableBytes - NumBytes;
2430
2431 // If this call requires more stack than we have available from
2432 // LowerFormalArguments, tell FrameLowering to reserve space for it.
2433 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff)
2434 AFI->setArgRegsSaveSize(-SPDiff);
2435 }
2436
2437 if (isSibCall) {
2438 // For sibling tail calls, memory operands are available in our caller's stack.
2439 NumBytes = 0;
2440 } else {
2441 // Adjust the stack pointer for the new arguments...
2442 // These operations are automatically eliminated by the prolog/epilog pass
2443 Chain = DAG.getCALLSEQ_START(Chain, isTailCall ? 0 : NumBytes, 0, dl);
2444 }
2445
2446 SDValue StackPtr =
2447 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
2448
2449 RegsToPassVector RegsToPass;
2450 SmallVector<SDValue, 8> MemOpChains;
2451
2452 // During a tail call, stores to the argument area must happen after all of
2453 // the function's incoming arguments have been loaded because they may alias.
2454 // This is done by folding in a TokenFactor from LowerFormalArguments, but
2455 // there's no point in doing so repeatedly so this tracks whether that's
2456 // happened yet.
2457 bool AfterFormalArgLoads = false;
2458
2459 // Walk the register/memloc assignments, inserting copies/loads. In the case
2460 // of tail call optimization, arguments are handled later.
2461 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2462 i != e;
2463 ++i, ++realArgIdx) {
2464 CCValAssign &VA = ArgLocs[i];
2465 SDValue Arg = OutVals[realArgIdx];
2466 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2467 bool isByVal = Flags.isByVal();
2468
2469 // Promote the value if needed.
2470 switch (VA.getLocInfo()) {
2471 default: llvm_unreachable("Unknown loc info!");
2472 case CCValAssign::Full: break;
2473 case CCValAssign::SExt:
2474 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
2475 break;
2476 case CCValAssign::ZExt:
2477 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
2478 break;
2479 case CCValAssign::AExt:
2480 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
2481 break;
2482 case CCValAssign::BCvt:
2483 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2484 break;
2485 }
2486
2487 if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
2488 Chain = DAG.getStackArgumentTokenFactor(Chain);
2489 AfterFormalArgLoads = true;
2490 }
2491
2492 // f16 arguments have their size extended to 4 bytes and passed as if they
2493 // had been copied to the LSBs of a 32-bit register.
2494 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2495 if (VA.needsCustom() &&
2496 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
2497 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
2498 } else {
2499 // f16 arguments could have been extended prior to argument lowering.
2500 // Mask them arguments if this is a CMSE nonsecure call.
2501 auto ArgVT = Outs[realArgIdx].ArgVT;
2502 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2503 auto LocBits = VA.getLocVT().getSizeInBits();
2504 auto MaskValue = APInt::getLowBitsSet(LocBits, ArgVT.getSizeInBits());
2505 SDValue Mask =
2506 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
2507 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
2508 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
2509 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2510 }
2511 }
2512
2513 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
2514 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
2515 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2516 DAG.getConstant(0, dl, MVT::i32));
2517 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2518 DAG.getConstant(1, dl, MVT::i32));
2519
2520 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2521 StackPtr, MemOpChains, isTailCall, SPDiff);
2522
2523 VA = ArgLocs[++i]; // skip ahead to next loc
2524 if (VA.isRegLoc()) {
2525 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2526 StackPtr, MemOpChains, isTailCall, SPDiff);
2527 } else {
2528 assert(VA.isMemLoc());
2529 SDValue DstAddr;
2530 MachinePointerInfo DstInfo;
2531 std::tie(DstAddr, DstInfo) =
2532 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2533 MemOpChains.push_back(DAG.getStore(Chain, dl, Op1, DstAddr, DstInfo));
2534 }
2535 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
2536 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2537 StackPtr, MemOpChains, isTailCall, SPDiff);
2538 } else if (VA.isRegLoc()) {
2539 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
2540 Outs[0].VT == MVT::i32) {
2541 assert(VA.getLocVT() == MVT::i32 &&
2542 "unexpected calling convention register assignment");
2543 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
2544 "unexpected use of 'returned'");
2545 isThisReturn = true;
2546 }
2547 const TargetOptions &Options = DAG.getTarget().Options;
2548 if (Options.EmitCallSiteInfo)
2549 CSInfo.emplace_back(VA.getLocReg(), i);
2550 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2551 } else if (isByVal) {
2552 assert(VA.isMemLoc());
2553 unsigned offset = 0;
2554
2555 // True if this byval aggregate will be split between registers
2556 // and memory.
2557 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2558 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2559
2560 if (CurByValIdx < ByValArgsCount) {
2561
2562 unsigned RegBegin, RegEnd;
2563 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2564
2565 EVT PtrVT =
2566 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2567 unsigned int i, j;
2568 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
2569 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
2570 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
2571 SDValue Load =
2572 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
2573 DAG.InferPtrAlign(AddArg));
2574 MemOpChains.push_back(Load.getValue(1));
2575 RegsToPass.push_back(std::make_pair(j, Load));
2576 }
2577
2578 // If parameter size outsides register area, "offset" value
2579 // helps us to calculate stack slot for remained part properly.
2580 offset = RegEnd - RegBegin;
2581
2582 CCInfo.nextInRegsParam();
2583 }
2584
2585 if (Flags.getByValSize() > 4*offset) {
2586 auto PtrVT = getPointerTy(DAG.getDataLayout());
2587 SDValue Dst;
2588 MachinePointerInfo DstInfo;
2589 std::tie(Dst, DstInfo) =
2590 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2591 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
2592 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
2593 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
2594 MVT::i32);
2595 SDValue AlignNode =
2596 DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
2597
2598 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
2599 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2600 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
2601 Ops));
2602 }
2603 } else {
2604 assert(VA.isMemLoc());
2605 SDValue DstAddr;
2606 MachinePointerInfo DstInfo;
2607 std::tie(DstAddr, DstInfo) =
2608 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2609
2610 SDValue Store = DAG.getStore(Chain, dl, Arg, DstAddr, DstInfo);
2611 MemOpChains.push_back(Store);
2612 }
2613 }
2614
2615 if (!MemOpChains.empty())
2616 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
2617
2618 // Build a sequence of copy-to-reg nodes chained together with token chain
2619 // and flag operands which copy the outgoing args into the appropriate regs.
2620 SDValue InFlag;
2621 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2622 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
2623 RegsToPass[i].second, InFlag);
2624 InFlag = Chain.getValue(1);
2625 }
2626
2627 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
2628 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
2629 // node so that legalize doesn't hack it.
2630 bool isDirect = false;
2631
2632 const TargetMachine &TM = getTargetMachine();
2633 const Module *Mod = MF.getFunction().getParent();
2634 const GlobalValue *GVal = nullptr;
2635 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
2636 GVal = G->getGlobal();
2637 bool isStub =
2638 !TM.shouldAssumeDSOLocal(*Mod, GVal) && Subtarget->isTargetMachO();
2639
2640 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
2641 bool isLocalARMFunc = false;
2642 auto PtrVt = getPointerTy(DAG.getDataLayout());
2643
2644 if (Subtarget->genLongCalls()) {
2645 assert((!isPositionIndependent() || Subtarget->isTargetWindows()) &&
2646 "long-calls codegen is not position independent!");
2647 // Handle a global address or an external symbol. If it's not one of
2648 // those, the target's already in a register, so we don't need to do
2649 // anything extra.
2650 if (isa<GlobalAddressSDNode>(Callee)) {
2651 // When generating execute-only code we use movw movt pair.
2652 // Currently execute-only is only available for architectures that
2653 // support movw movt, so we are safe to assume that.
2654 if (Subtarget->genExecuteOnly()) {
2655 assert(Subtarget->useMovt() &&
2656 "long-calls with execute-only requires movt and movw!");
2657 ++NumMovwMovt;
2658 Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
2659 DAG.getTargetGlobalAddress(GVal, dl, PtrVt));
2660 } else {
2661 // Create a constant pool entry for the callee address
2662 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2663 ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
2664 GVal, ARMPCLabelIndex, ARMCP::CPValue, 0);
2665
2666 // Get the address of the callee into a register
2667 SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2668 Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2669 Callee = DAG.getLoad(
2670 PtrVt, dl, DAG.getEntryNode(), Addr,
2671 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2672 }
2673 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) {
2674 const char *Sym = S->getSymbol();
2675
2676 // When generating execute-only code we use movw movt pair.
2677 // Currently execute-only is only available for architectures that
2678 // support movw movt, so we are safe to assume that.
2679 if (Subtarget->genExecuteOnly()) {
2680 assert(Subtarget->useMovt() &&
2681 "long-calls with execute-only requires movt and movw!");
2682 ++NumMovwMovt;
2683 Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
2684 DAG.getTargetGlobalAddress(GVal, dl, PtrVt));
2685 } else {
2686 // Create a constant pool entry for the callee address
2687 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2688 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2689 *DAG.getContext(), Sym, ARMPCLabelIndex, 0);
2690
2691 // Get the address of the callee into a register
2692 SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2693 Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2694 Callee = DAG.getLoad(
2695 PtrVt, dl, DAG.getEntryNode(), Addr,
2696 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2697 }
2698 }
2699 } else if (isa<GlobalAddressSDNode>(Callee)) {
2700 if (!PreferIndirect) {
2701 isDirect = true;
2702 bool isDef = GVal->isStrongDefinitionForLinker();
2703
2704 // ARM call to a local ARM function is predicable.
2705 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
2706 // tBX takes a register source operand.
2707 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2708 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?");
2709 Callee = DAG.getNode(
2710 ARMISD::WrapperPIC, dl, PtrVt,
2711 DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, ARMII::MO_NONLAZY));
2712 Callee = DAG.getLoad(
2713 PtrVt, dl, DAG.getEntryNode(), Callee,
2714 MachinePointerInfo::getGOT(DAG.getMachineFunction()), MaybeAlign(),
2715 MachineMemOperand::MODereferenceable |
2716 MachineMemOperand::MOInvariant);
2717 } else if (Subtarget->isTargetCOFF()) {
2718 assert(Subtarget->isTargetWindows() &&
2719 "Windows is the only supported COFF target");
2720 unsigned TargetFlags = ARMII::MO_NO_FLAG;
2721 if (GVal->hasDLLImportStorageClass())
2722 TargetFlags = ARMII::MO_DLLIMPORT;
2723 else if (!TM.shouldAssumeDSOLocal(*GVal->getParent(), GVal))
2724 TargetFlags = ARMII::MO_COFFSTUB;
2725 Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, /*offset=*/0,
2726 TargetFlags);
2727 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
2728 Callee =
2729 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
2730 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
2731 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2732 } else {
2733 Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, 0);
2734 }
2735 }
2736 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2737 isDirect = true;
2738 // tBX takes a register source operand.
2739 const char *Sym = S->getSymbol();
2740 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2741 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2742 ARMConstantPoolValue *CPV =
2743 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
2744 ARMPCLabelIndex, 4);
2745 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2746 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2747 Callee = DAG.getLoad(
2748 PtrVt, dl, DAG.getEntryNode(), CPAddr,
2749 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2750 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2751 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
2752 } else {
2753 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0);
2754 }
2755 }
2756
2757 if (isCmseNSCall) {
2758 assert(!isARMFunc && !isDirect &&
2759 "Cannot handle call to ARM function or direct call");
2760 if (NumBytes > 0) {
2761 DiagnosticInfoUnsupported Diag(DAG.getMachineFunction().getFunction(),
2762 "call to non-secure function would "
2763 "require passing arguments on stack",
2764 dl.getDebugLoc());
2765 DAG.getContext()->diagnose(Diag);
2766 }
2767 if (isStructRet) {
2768 DiagnosticInfoUnsupported Diag(
2769 DAG.getMachineFunction().getFunction(),
2770 "call to non-secure function would return value through pointer",
2771 dl.getDebugLoc());
2772 DAG.getContext()->diagnose(Diag);
2773 }
2774 }
2775
2776 // FIXME: handle tail calls differently.
2777 unsigned CallOpc;
2778 if (Subtarget->isThumb()) {
2779 if (GuardWithBTI)
2780 CallOpc = ARMISD::t2CALL_BTI;
2781 else if (isCmseNSCall)
2782 CallOpc = ARMISD::tSECALL;
2783 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2784 CallOpc = ARMISD::CALL_NOLINK;
2785 else
2786 CallOpc = ARMISD::CALL;
2787 } else {
2788 if (!isDirect && !Subtarget->hasV5TOps())
2789 CallOpc = ARMISD::CALL_NOLINK;
2790 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2791 // Emit regular call when code size is the priority
2792 !Subtarget->hasMinSize())
2793 // "mov lr, pc; b _foo" to avoid confusing the RSP
2794 CallOpc = ARMISD::CALL_NOLINK;
2795 else
2796 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
2797 }
2798
2799 // We don't usually want to end the call-sequence here because we would tidy
2800 // the frame up *after* the call, however in the ABI-changing tail-call case
2801 // we've carefully laid out the parameters so that when sp is reset they'll be
2802 // in the correct location.
2803 if (isTailCall && !isSibCall) {
2804 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InFlag, dl);
2805 InFlag = Chain.getValue(1);
2806 }
2807
2808 std::vector<SDValue> Ops;
2809 Ops.push_back(Chain);
2810 Ops.push_back(Callee);
2811
2812 if (isTailCall) {
2813 Ops.push_back(DAG.getTargetConstant(SPDiff, dl, MVT::i32));
2814 }
2815
2816 // Add argument registers to the end of the list so that they are known live
2817 // into the call.
2818 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2819 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
2820 RegsToPass[i].second.getValueType()));
2821
2822 // Add a register mask operand representing the call-preserved registers.
2823 const uint32_t *Mask;
2824 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
2825 if (isThisReturn) {
2826 // For 'this' returns, use the R0-preserving mask if applicable
2827 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
2828 if (!Mask) {
2829 // Set isThisReturn to false if the calling convention is not one that
2830 // allows 'returned' to be modeled in this way, so LowerCallResult does
2831 // not try to pass 'this' straight through
2832 isThisReturn = false;
2833 Mask = ARI->getCallPreservedMask(MF, CallConv);
2834 }
2835 } else
2836 Mask = ARI->getCallPreservedMask(MF, CallConv);
2837
2838 assert(Mask && "Missing call preserved mask for calling convention");
2839 Ops.push_back(DAG.getRegisterMask(Mask));
2840
2841 if (InFlag.getNode())
2842 Ops.push_back(InFlag);
2843
2844 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2845 if (isTailCall) {
2846 MF.getFrameInfo().setHasTailCall();
2847 SDValue Ret = DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops);
2848 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
2849 return Ret;
2850 }
2851
2852 // Returns a chain and a flag for retval copy to use.
2853 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
2854 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
2855 InFlag = Chain.getValue(1);
2856 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
2857
2858 // If we're guaranteeing tail-calls will be honoured, the callee must
2859 // pop its own argument stack on return. But this call is *not* a tail call so
2860 // we need to undo that after it returns to restore the status-quo.
2861 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
2862 uint64_t CalleePopBytes =
2863 canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1ULL;
2864
2865 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InFlag, dl);
2866 if (!Ins.empty())
2867 InFlag = Chain.getValue(1);
2868
2869 // Handle result values, copying them out of physregs into vregs that we
2870 // return.
2871 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG,
2872 InVals, isThisReturn,
2873 isThisReturn ? OutVals[0] : SDValue());
2874 }
2875
2876 /// HandleByVal - Every parameter *after* a byval parameter is passed
2877 /// on the stack. Remember the next parameter register to allocate,
2878 /// and then confiscate the rest of the parameter registers to insure
2879 /// this.
HandleByVal(CCState * State,unsigned & Size,Align Alignment) const2880 void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
2881 Align Alignment) const {
2882 // Byval (as with any stack) slots are always at least 4 byte aligned.
2883 Alignment = std::max(Alignment, Align(4));
2884
2885 unsigned Reg = State->AllocateReg(GPRArgRegs);
2886 if (!Reg)
2887 return;
2888
2889 unsigned AlignInRegs = Alignment.value() / 4;
2890 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
2891 for (unsigned i = 0; i < Waste; ++i)
2892 Reg = State->AllocateReg(GPRArgRegs);
2893
2894 if (!Reg)
2895 return;
2896
2897 unsigned Excess = 4 * (ARM::R4 - Reg);
2898
2899 // Special case when NSAA != SP and parameter size greater than size of
2900 // all remained GPR regs. In that case we can't split parameter, we must
2901 // send it to stack. We also must set NCRN to R4, so waste all
2902 // remained registers.
2903 const unsigned NSAAOffset = State->getNextStackOffset();
2904 if (NSAAOffset != 0 && Size > Excess) {
2905 while (State->AllocateReg(GPRArgRegs))
2906 ;
2907 return;
2908 }
2909
2910 // First register for byval parameter is the first register that wasn't
2911 // allocated before this method call, so it would be "reg".
2912 // If parameter is small enough to be saved in range [reg, r4), then
2913 // the end (first after last) register would be reg + param-size-in-regs,
2914 // else parameter would be splitted between registers and stack,
2915 // end register would be r4 in this case.
2916 unsigned ByValRegBegin = Reg;
2917 unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4);
2918 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd);
2919 // Note, first register is allocated in the beginning of function already,
2920 // allocate remained amount of registers we need.
2921 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2922 State->AllocateReg(GPRArgRegs);
2923 // A byval parameter that is split between registers and memory needs its
2924 // size truncated here.
2925 // In the case where the entire structure fits in registers, we set the
2926 // size in memory to zero.
2927 Size = std::max<int>(Size - Excess, 0);
2928 }
2929
2930 /// MatchingStackOffset - Return true if the given stack call argument is
2931 /// already available in the same position (relatively) of the caller's
2932 /// incoming argument stack.
2933 static
MatchingStackOffset(SDValue Arg,unsigned Offset,ISD::ArgFlagsTy Flags,MachineFrameInfo & MFI,const MachineRegisterInfo * MRI,const TargetInstrInfo * TII)2934 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
2935 MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,
2936 const TargetInstrInfo *TII) {
2937 unsigned Bytes = Arg.getValueSizeInBits() / 8;
2938 int FI = std::numeric_limits<int>::max();
2939 if (Arg.getOpcode() == ISD::CopyFromReg) {
2940 Register VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
2941 if (!VR.isVirtual())
2942 return false;
2943 MachineInstr *Def = MRI->getVRegDef(VR);
2944 if (!Def)
2945 return false;
2946 if (!Flags.isByVal()) {
2947 if (!TII->isLoadFromStackSlot(*Def, FI))
2948 return false;
2949 } else {
2950 return false;
2951 }
2952 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2953 if (Flags.isByVal())
2954 // ByVal argument is passed in as a pointer but it's now being
2955 // dereferenced. e.g.
2956 // define @foo(%struct.X* %A) {
2957 // tail call @bar(%struct.X* byval %A)
2958 // }
2959 return false;
2960 SDValue Ptr = Ld->getBasePtr();
2961 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
2962 if (!FINode)
2963 return false;
2964 FI = FINode->getIndex();
2965 } else
2966 return false;
2967
2968 assert(FI != std::numeric_limits<int>::max());
2969 if (!MFI.isFixedObjectIndex(FI))
2970 return false;
2971 return Offset == MFI.getObjectOffset(FI) && Bytes == MFI.getObjectSize(FI);
2972 }
2973
2974 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
2975 /// for tail call optimization. Targets which want to do tail call
2976 /// optimization should implement this function.
IsEligibleForTailCallOptimization(SDValue Callee,CallingConv::ID CalleeCC,bool isVarArg,bool isCalleeStructRet,bool isCallerStructRet,const SmallVectorImpl<ISD::OutputArg> & Outs,const SmallVectorImpl<SDValue> & OutVals,const SmallVectorImpl<ISD::InputArg> & Ins,SelectionDAG & DAG,const bool isIndirect) const2977 bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2978 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
2979 bool isCalleeStructRet, bool isCallerStructRet,
2980 const SmallVectorImpl<ISD::OutputArg> &Outs,
2981 const SmallVectorImpl<SDValue> &OutVals,
2982 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG,
2983 const bool isIndirect) const {
2984 MachineFunction &MF = DAG.getMachineFunction();
2985 const Function &CallerF = MF.getFunction();
2986 CallingConv::ID CallerCC = CallerF.getCallingConv();
2987
2988 assert(Subtarget->supportsTailCall());
2989
2990 // Indirect tail calls cannot be optimized for Thumb1 if the args
2991 // to the call take up r0-r3. The reason is that there are no legal registers
2992 // left to hold the pointer to the function to be called.
2993 // Similarly, if the function uses return address sign and authentication,
2994 // r12 is needed to hold the PAC and is not available to hold the callee
2995 // address.
2996 if (Outs.size() >= 4 &&
2997 (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect)) {
2998 if (Subtarget->isThumb1Only())
2999 return false;
3000 // Conservatively assume the function spills LR.
3001 if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(true))
3002 return false;
3003 }
3004
3005 // Look for obvious safe cases to perform tail call optimization that do not
3006 // require ABI changes. This is what gcc calls sibcall.
3007
3008 // Exception-handling functions need a special set of instructions to indicate
3009 // a return to the hardware. Tail-calling another function would probably
3010 // break this.
3011 if (CallerF.hasFnAttribute("interrupt"))
3012 return false;
3013
3014 if (canGuaranteeTCO(CalleeCC, getTargetMachine().Options.GuaranteedTailCallOpt))
3015 return CalleeCC == CallerCC;
3016
3017 // Also avoid sibcall optimization if either caller or callee uses struct
3018 // return semantics.
3019 if (isCalleeStructRet || isCallerStructRet)
3020 return false;
3021
3022 // Externally-defined functions with weak linkage should not be
3023 // tail-called on ARM when the OS does not support dynamic
3024 // pre-emption of symbols, as the AAELF spec requires normal calls
3025 // to undefined weak functions to be replaced with a NOP or jump to the
3026 // next instruction. The behaviour of branch instructions in this
3027 // situation (as used for tail calls) is implementation-defined, so we
3028 // cannot rely on the linker replacing the tail call with a return.
3029 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3030 const GlobalValue *GV = G->getGlobal();
3031 const Triple &TT = getTargetMachine().getTargetTriple();
3032 if (GV->hasExternalWeakLinkage() &&
3033 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
3034 return false;
3035 }
3036
3037 // Check that the call results are passed in the same way.
3038 LLVMContext &C = *DAG.getContext();
3039 if (!CCState::resultsCompatible(
3040 getEffectiveCallingConv(CalleeCC, isVarArg),
3041 getEffectiveCallingConv(CallerCC, CallerF.isVarArg()), MF, C, Ins,
3042 CCAssignFnForReturn(CalleeCC, isVarArg),
3043 CCAssignFnForReturn(CallerCC, CallerF.isVarArg())))
3044 return false;
3045 // The callee has to preserve all registers the caller needs to preserve.
3046 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
3047 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
3048 if (CalleeCC != CallerCC) {
3049 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
3050 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3051 return false;
3052 }
3053
3054 // If Caller's vararg or byval argument has been split between registers and
3055 // stack, do not perform tail call, since part of the argument is in caller's
3056 // local frame.
3057 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
3058 if (AFI_Caller->getArgRegsSaveSize())
3059 return false;
3060
3061 // If the callee takes no arguments then go on to check the results of the
3062 // call.
3063 if (!Outs.empty()) {
3064 // Check if stack adjustment is needed. For now, do not do this if any
3065 // argument is passed on the stack.
3066 SmallVector<CCValAssign, 16> ArgLocs;
3067 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
3068 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
3069 if (CCInfo.getNextStackOffset()) {
3070 // Check if the arguments are already laid out in the right way as
3071 // the caller's fixed stack objects.
3072 MachineFrameInfo &MFI = MF.getFrameInfo();
3073 const MachineRegisterInfo *MRI = &MF.getRegInfo();
3074 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3075 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
3076 i != e;
3077 ++i, ++realArgIdx) {
3078 CCValAssign &VA = ArgLocs[i];
3079 EVT RegVT = VA.getLocVT();
3080 SDValue Arg = OutVals[realArgIdx];
3081 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
3082 if (VA.getLocInfo() == CCValAssign::Indirect)
3083 return false;
3084 if (VA.needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3085 // f64 and vector types are split into multiple registers or
3086 // register/stack-slot combinations. The types will not match
3087 // the registers; give up on memory f64 refs until we figure
3088 // out what to do about this.
3089 if (!VA.isRegLoc())
3090 return false;
3091 if (!ArgLocs[++i].isRegLoc())
3092 return false;
3093 if (RegVT == MVT::v2f64) {
3094 if (!ArgLocs[++i].isRegLoc())
3095 return false;
3096 if (!ArgLocs[++i].isRegLoc())
3097 return false;
3098 }
3099 } else if (!VA.isRegLoc()) {
3100 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
3101 MFI, MRI, TII))
3102 return false;
3103 }
3104 }
3105 }
3106
3107 const MachineRegisterInfo &MRI = MF.getRegInfo();
3108 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
3109 return false;
3110 }
3111
3112 return true;
3113 }
3114
3115 bool
CanLowerReturn(CallingConv::ID CallConv,MachineFunction & MF,bool isVarArg,const SmallVectorImpl<ISD::OutputArg> & Outs,LLVMContext & Context) const3116 ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
3117 MachineFunction &MF, bool isVarArg,
3118 const SmallVectorImpl<ISD::OutputArg> &Outs,
3119 LLVMContext &Context) const {
3120 SmallVector<CCValAssign, 16> RVLocs;
3121 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3122 return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
3123 }
3124
LowerInterruptReturn(SmallVectorImpl<SDValue> & RetOps,const SDLoc & DL,SelectionDAG & DAG)3125 static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
3126 const SDLoc &DL, SelectionDAG &DAG) {
3127 const MachineFunction &MF = DAG.getMachineFunction();
3128 const Function &F = MF.getFunction();
3129
3130 StringRef IntKind = F.getFnAttribute("interrupt").getValueAsString();
3131
3132 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
3133 // version of the "preferred return address". These offsets affect the return
3134 // instruction if this is a return from PL1 without hypervisor extensions.
3135 // IRQ/FIQ: +4 "subs pc, lr, #4"
3136 // SWI: 0 "subs pc, lr, #0"
3137 // ABORT: +4 "subs pc, lr, #4"
3138 // UNDEF: +4/+2 "subs pc, lr, #0"
3139 // UNDEF varies depending on where the exception came from ARM or Thumb
3140 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
3141
3142 int64_t LROffset;
3143 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
3144 IntKind == "ABORT")
3145 LROffset = 4;
3146 else if (IntKind == "SWI" || IntKind == "UNDEF")
3147 LROffset = 0;
3148 else
3149 report_fatal_error("Unsupported interrupt attribute. If present, value "
3150 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3151
3152 RetOps.insert(RetOps.begin() + 1,
3153 DAG.getConstant(LROffset, DL, MVT::i32, false));
3154
3155 return DAG.getNode(ARMISD::INTRET_FLAG, DL, MVT::Other, RetOps);
3156 }
3157
3158 SDValue
LowerReturn(SDValue Chain,CallingConv::ID CallConv,bool isVarArg,const SmallVectorImpl<ISD::OutputArg> & Outs,const SmallVectorImpl<SDValue> & OutVals,const SDLoc & dl,SelectionDAG & DAG) const3159 ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3160 bool isVarArg,
3161 const SmallVectorImpl<ISD::OutputArg> &Outs,
3162 const SmallVectorImpl<SDValue> &OutVals,
3163 const SDLoc &dl, SelectionDAG &DAG) const {
3164 // CCValAssign - represent the assignment of the return value to a location.
3165 SmallVector<CCValAssign, 16> RVLocs;
3166
3167 // CCState - Info about the registers and stack slots.
3168 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3169 *DAG.getContext());
3170
3171 // Analyze outgoing return values.
3172 CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
3173
3174 SDValue Flag;
3175 SmallVector<SDValue, 4> RetOps;
3176 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
3177 bool isLittleEndian = Subtarget->isLittle();
3178
3179 MachineFunction &MF = DAG.getMachineFunction();
3180 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3181 AFI->setReturnRegsCount(RVLocs.size());
3182
3183 // Report error if cmse entry function returns structure through first ptr arg.
3184 if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) {
3185 // Note: using an empty SDLoc(), as the first line of the function is a
3186 // better place to report than the last line.
3187 DiagnosticInfoUnsupported Diag(
3188 DAG.getMachineFunction().getFunction(),
3189 "secure entry function would return value through pointer",
3190 SDLoc().getDebugLoc());
3191 DAG.getContext()->diagnose(Diag);
3192 }
3193
3194 // Copy the result values into the output registers.
3195 for (unsigned i = 0, realRVLocIdx = 0;
3196 i != RVLocs.size();
3197 ++i, ++realRVLocIdx) {
3198 CCValAssign &VA = RVLocs[i];
3199 assert(VA.isRegLoc() && "Can only return in registers!");
3200
3201 SDValue Arg = OutVals[realRVLocIdx];
3202 bool ReturnF16 = false;
3203
3204 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
3205 // Half-precision return values can be returned like this:
3206 //
3207 // t11 f16 = fadd ...
3208 // t12: i16 = bitcast t11
3209 // t13: i32 = zero_extend t12
3210 // t14: f32 = bitcast t13 <~~~~~~~ Arg
3211 //
3212 // to avoid code generation for bitcasts, we simply set Arg to the node
3213 // that produces the f16 value, t11 in this case.
3214 //
3215 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) {
3216 SDValue ZE = Arg.getOperand(0);
3217 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) {
3218 SDValue BC = ZE.getOperand(0);
3219 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) {
3220 Arg = BC.getOperand(0);
3221 ReturnF16 = true;
3222 }
3223 }
3224 }
3225 }
3226
3227 switch (VA.getLocInfo()) {
3228 default: llvm_unreachable("Unknown loc info!");
3229 case CCValAssign::Full: break;
3230 case CCValAssign::BCvt:
3231 if (!ReturnF16)
3232 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3233 break;
3234 }
3235
3236 // Mask f16 arguments if this is a CMSE nonsecure entry.
3237 auto RetVT = Outs[realRVLocIdx].ArgVT;
3238 if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
3239 if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
3240 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
3241 } else {
3242 auto LocBits = VA.getLocVT().getSizeInBits();
3243 auto MaskValue = APInt::getLowBitsSet(LocBits, RetVT.getSizeInBits());
3244 SDValue Mask =
3245 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
3246 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
3247 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
3248 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3249 }
3250 }
3251
3252 if (VA.needsCustom() &&
3253 (VA.getLocVT() == MVT::v2f64 || VA.getLocVT() == MVT::f64)) {
3254 if (VA.getLocVT() == MVT::v2f64) {
3255 // Extract the first half and return it in two registers.
3256 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3257 DAG.getConstant(0, dl, MVT::i32));
3258 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
3259 DAG.getVTList(MVT::i32, MVT::i32), Half);
3260
3261 Chain =
3262 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3263 HalfGPRs.getValue(isLittleEndian ? 0 : 1), Flag);
3264 Flag = Chain.getValue(1);
3265 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3266 VA = RVLocs[++i]; // skip ahead to next loc
3267 Chain =
3268 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3269 HalfGPRs.getValue(isLittleEndian ? 1 : 0), Flag);
3270 Flag = Chain.getValue(1);
3271 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3272 VA = RVLocs[++i]; // skip ahead to next loc
3273
3274 // Extract the 2nd half and fall through to handle it as an f64 value.
3275 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3276 DAG.getConstant(1, dl, MVT::i32));
3277 }
3278 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
3279 // available.
3280 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
3281 DAG.getVTList(MVT::i32, MVT::i32), Arg);
3282 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3283 fmrrd.getValue(isLittleEndian ? 0 : 1), Flag);
3284 Flag = Chain.getValue(1);
3285 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3286 VA = RVLocs[++i]; // skip ahead to next loc
3287 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3288 fmrrd.getValue(isLittleEndian ? 1 : 0), Flag);
3289 } else
3290 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
3291
3292 // Guarantee that all emitted copies are
3293 // stuck together, avoiding something bad.
3294 Flag = Chain.getValue(1);
3295 RetOps.push_back(DAG.getRegister(
3296 VA.getLocReg(), ReturnF16 ? Arg.getValueType() : VA.getLocVT()));
3297 }
3298 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
3299 const MCPhysReg *I =
3300 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
3301 if (I) {
3302 for (; *I; ++I) {
3303 if (ARM::GPRRegClass.contains(*I))
3304 RetOps.push_back(DAG.getRegister(*I, MVT::i32));
3305 else if (ARM::DPRRegClass.contains(*I))
3306 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
3307 else
3308 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3309 }
3310 }
3311
3312 // Update chain and glue.
3313 RetOps[0] = Chain;
3314 if (Flag.getNode())
3315 RetOps.push_back(Flag);
3316
3317 // CPUs which aren't M-class use a special sequence to return from
3318 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
3319 // though we use "subs pc, lr, #N").
3320 //
3321 // M-class CPUs actually use a normal return sequence with a special
3322 // (hardware-provided) value in LR, so the normal code path works.
3323 if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt") &&
3324 !Subtarget->isMClass()) {
3325 if (Subtarget->isThumb1Only())
3326 report_fatal_error("interrupt attribute is not supported in Thumb1");
3327 return LowerInterruptReturn(RetOps, dl, DAG);
3328 }
3329
3330 ARMISD::NodeType RetNode = AFI->isCmseNSEntryFunction() ? ARMISD::SERET_FLAG :
3331 ARMISD::RET_FLAG;
3332 return DAG.getNode(RetNode, dl, MVT::Other, RetOps);
3333 }
3334
isUsedByReturnOnly(SDNode * N,SDValue & Chain) const3335 bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
3336 if (N->getNumValues() != 1)
3337 return false;
3338 if (!N->hasNUsesOfValue(1, 0))
3339 return false;
3340
3341 SDValue TCChain = Chain;
3342 SDNode *Copy = *N->use_begin();
3343 if (Copy->getOpcode() == ISD::CopyToReg) {
3344 // If the copy has a glue operand, we conservatively assume it isn't safe to
3345 // perform a tail call.
3346 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3347 return false;
3348 TCChain = Copy->getOperand(0);
3349 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
3350 SDNode *VMov = Copy;
3351 // f64 returned in a pair of GPRs.
3352 SmallPtrSet<SDNode*, 2> Copies;
3353 for (SDNode *U : VMov->uses()) {
3354 if (U->getOpcode() != ISD::CopyToReg)
3355 return false;
3356 Copies.insert(U);
3357 }
3358 if (Copies.size() > 2)
3359 return false;
3360
3361 for (SDNode *U : VMov->uses()) {
3362 SDValue UseChain = U->getOperand(0);
3363 if (Copies.count(UseChain.getNode()))
3364 // Second CopyToReg
3365 Copy = U;
3366 else {
3367 // We are at the top of this chain.
3368 // If the copy has a glue operand, we conservatively assume it
3369 // isn't safe to perform a tail call.
3370 if (U->getOperand(U->getNumOperands() - 1).getValueType() == MVT::Glue)
3371 return false;
3372 // First CopyToReg
3373 TCChain = UseChain;
3374 }
3375 }
3376 } else if (Copy->getOpcode() == ISD::BITCAST) {
3377 // f32 returned in a single GPR.
3378 if (!Copy->hasOneUse())
3379 return false;
3380 Copy = *Copy->use_begin();
3381 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0))
3382 return false;
3383 // If the copy has a glue operand, we conservatively assume it isn't safe to
3384 // perform a tail call.
3385 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3386 return false;
3387 TCChain = Copy->getOperand(0);
3388 } else {
3389 return false;
3390 }
3391
3392 bool HasRet = false;
3393 for (const SDNode *U : Copy->uses()) {
3394 if (U->getOpcode() != ARMISD::RET_FLAG &&
3395 U->getOpcode() != ARMISD::INTRET_FLAG)
3396 return false;
3397 HasRet = true;
3398 }
3399
3400 if (!HasRet)
3401 return false;
3402
3403 Chain = TCChain;
3404 return true;
3405 }
3406
mayBeEmittedAsTailCall(const CallInst * CI) const3407 bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3408 if (!Subtarget->supportsTailCall())
3409 return false;
3410
3411 if (!CI->isTailCall())
3412 return false;
3413
3414 return true;
3415 }
3416
3417 // Trying to write a 64 bit value so need to split into two 32 bit values first,
3418 // and pass the lower and high parts through.
LowerWRITE_REGISTER(SDValue Op,SelectionDAG & DAG)3419 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
3420 SDLoc DL(Op);
3421 SDValue WriteValue = Op->getOperand(2);
3422
3423 // This function is only supposed to be called for i64 type argument.
3424 assert(WriteValue.getValueType() == MVT::i64
3425 && "LowerWRITE_REGISTER called for non-i64 type argument.");
3426
3427 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
3428 DAG.getConstant(0, DL, MVT::i32));
3429 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
3430 DAG.getConstant(1, DL, MVT::i32));
3431 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
3432 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
3433 }
3434
3435 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
3436 // their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
3437 // one of the above mentioned nodes. It has to be wrapped because otherwise
3438 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
3439 // be used to form addressing mode. These wrapped nodes will be selected
3440 // into MOVi.
LowerConstantPool(SDValue Op,SelectionDAG & DAG) const3441 SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
3442 SelectionDAG &DAG) const {
3443 EVT PtrVT = Op.getValueType();
3444 // FIXME there is no actual debug info here
3445 SDLoc dl(Op);
3446 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
3447 SDValue Res;
3448
3449 // When generating execute-only code Constant Pools must be promoted to the
3450 // global data section. It's a bit ugly that we can't share them across basic
3451 // blocks, but this way we guarantee that execute-only behaves correct with
3452 // position-independent addressing modes.
3453 if (Subtarget->genExecuteOnly()) {
3454 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
3455 auto T = const_cast<Type*>(CP->getType());
3456 auto C = const_cast<Constant*>(CP->getConstVal());
3457 auto M = const_cast<Module*>(DAG.getMachineFunction().
3458 getFunction().getParent());
3459 auto GV = new GlobalVariable(
3460 *M, T, /*isConstant=*/true, GlobalVariable::InternalLinkage, C,
3461 Twine(DAG.getDataLayout().getPrivateGlobalPrefix()) + "CP" +
3462 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
3463 Twine(AFI->createPICLabelUId())
3464 );
3465 SDValue GA = DAG.getTargetGlobalAddress(dyn_cast<GlobalValue>(GV),
3466 dl, PtrVT);
3467 return LowerGlobalAddress(GA, DAG);
3468 }
3469
3470 // The 16-bit ADR instruction can only encode offsets that are multiples of 4,
3471 // so we need to align to at least 4 bytes when we don't have 32-bit ADR.
3472 Align CPAlign = CP->getAlign();
3473 if (Subtarget->isThumb1Only())
3474 CPAlign = std::max(CPAlign, Align(4));
3475 if (CP->isMachineConstantPoolEntry())
3476 Res =
3477 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CPAlign);
3478 else
3479 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CPAlign);
3480 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
3481 }
3482
getJumpTableEncoding() const3483 unsigned ARMTargetLowering::getJumpTableEncoding() const {
3484 return MachineJumpTableInfo::EK_Inline;
3485 }
3486
LowerBlockAddress(SDValue Op,SelectionDAG & DAG) const3487 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3488 SelectionDAG &DAG) const {
3489 MachineFunction &MF = DAG.getMachineFunction();
3490 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3491 unsigned ARMPCLabelIndex = 0;
3492 SDLoc DL(Op);
3493 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3494 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
3495 SDValue CPAddr;
3496 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
3497 if (!IsPositionIndependent) {
3498 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, Align(4));
3499 } else {
3500 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3501 ARMPCLabelIndex = AFI->createPICLabelUId();
3502 ARMConstantPoolValue *CPV =
3503 ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex,
3504 ARMCP::CPBlockAddress, PCAdj);
3505 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3506 }
3507 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
3508 SDValue Result = DAG.getLoad(
3509 PtrVT, DL, DAG.getEntryNode(), CPAddr,
3510 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3511 if (!IsPositionIndependent)
3512 return Result;
3513 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
3514 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
3515 }
3516
3517 /// Convert a TLS address reference into the correct sequence of loads
3518 /// and calls to compute the variable's address for Darwin, and return an
3519 /// SDValue containing the final node.
3520
3521 /// Darwin only has one TLS scheme which must be capable of dealing with the
3522 /// fully general situation, in the worst case. This means:
3523 /// + "extern __thread" declaration.
3524 /// + Defined in a possibly unknown dynamic library.
3525 ///
3526 /// The general system is that each __thread variable has a [3 x i32] descriptor
3527 /// which contains information used by the runtime to calculate the address. The
3528 /// only part of this the compiler needs to know about is the first word, which
3529 /// contains a function pointer that must be called with the address of the
3530 /// entire descriptor in "r0".
3531 ///
3532 /// Since this descriptor may be in a different unit, in general access must
3533 /// proceed along the usual ARM rules. A common sequence to produce is:
3534 ///
3535 /// movw rT1, :lower16:_var$non_lazy_ptr
3536 /// movt rT1, :upper16:_var$non_lazy_ptr
3537 /// ldr r0, [rT1]
3538 /// ldr rT2, [r0]
3539 /// blx rT2
3540 /// [...address now in r0...]
3541 SDValue
LowerGlobalTLSAddressDarwin(SDValue Op,SelectionDAG & DAG) const3542 ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3543 SelectionDAG &DAG) const {
3544 assert(Subtarget->isTargetDarwin() &&
3545 "This function expects a Darwin target");
3546 SDLoc DL(Op);
3547
3548 // First step is to get the address of the actua global symbol. This is where
3549 // the TLS descriptor lives.
3550 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3551
3552 // The first entry in the descriptor is a function pointer that we must call
3553 // to obtain the address of the variable.
3554 SDValue Chain = DAG.getEntryNode();
3555 SDValue FuncTLVGet = DAG.getLoad(
3556 MVT::i32, DL, Chain, DescAddr,
3557 MachinePointerInfo::getGOT(DAG.getMachineFunction()), Align(4),
3558 MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable |
3559 MachineMemOperand::MOInvariant);
3560 Chain = FuncTLVGet.getValue(1);
3561
3562 MachineFunction &F = DAG.getMachineFunction();
3563 MachineFrameInfo &MFI = F.getFrameInfo();
3564 MFI.setAdjustsStack(true);
3565
3566 // TLS calls preserve all registers except those that absolutely must be
3567 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
3568 // silly).
3569 auto TRI =
3570 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo();
3571 auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
3572 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction());
3573
3574 // Finally, we can make the call. This is just a degenerate version of a
3575 // normal AArch64 call node: r0 takes the address of the descriptor, and
3576 // returns the address of the variable in this thread.
3577 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue());
3578 Chain =
3579 DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
3580 Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32),
3581 DAG.getRegisterMask(Mask), Chain.getValue(1));
3582 return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1));
3583 }
3584
3585 SDValue
LowerGlobalTLSAddressWindows(SDValue Op,SelectionDAG & DAG) const3586 ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3587 SelectionDAG &DAG) const {
3588 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering");
3589
3590 SDValue Chain = DAG.getEntryNode();
3591 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3592 SDLoc DL(Op);
3593
3594 // Load the current TEB (thread environment block)
3595 SDValue Ops[] = {Chain,
3596 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
3597 DAG.getTargetConstant(15, DL, MVT::i32),
3598 DAG.getTargetConstant(0, DL, MVT::i32),
3599 DAG.getTargetConstant(13, DL, MVT::i32),
3600 DAG.getTargetConstant(0, DL, MVT::i32),
3601 DAG.getTargetConstant(2, DL, MVT::i32)};
3602 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
3603 DAG.getVTList(MVT::i32, MVT::Other), Ops);
3604
3605 SDValue TEB = CurrentTEB.getValue(0);
3606 Chain = CurrentTEB.getValue(1);
3607
3608 // Load the ThreadLocalStoragePointer from the TEB
3609 // A pointer to the TLS array is located at offset 0x2c from the TEB.
3610 SDValue TLSArray =
3611 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL));
3612 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
3613
3614 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
3615 // offset into the TLSArray.
3616
3617 // Load the TLS index from the C runtime
3618 SDValue TLSIndex =
3619 DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG);
3620 TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex);
3621 TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo());
3622
3623 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
3624 DAG.getConstant(2, DL, MVT::i32));
3625 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
3626 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
3627 MachinePointerInfo());
3628
3629 // Get the offset of the start of the .tls section (section base)
3630 const auto *GA = cast<GlobalAddressSDNode>(Op);
3631 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL);
3632 SDValue Offset = DAG.getLoad(
3633 PtrVT, DL, Chain,
3634 DAG.getNode(ARMISD::Wrapper, DL, MVT::i32,
3635 DAG.getTargetConstantPool(CPV, PtrVT, Align(4))),
3636 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3637
3638 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset);
3639 }
3640
3641 // Lower ISD::GlobalTLSAddress using the "general dynamic" model
3642 SDValue
LowerToTLSGeneralDynamicModel(GlobalAddressSDNode * GA,SelectionDAG & DAG) const3643 ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
3644 SelectionDAG &DAG) const {
3645 SDLoc dl(GA);
3646 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3647 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3648 MachineFunction &MF = DAG.getMachineFunction();
3649 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3650 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3651 ARMConstantPoolValue *CPV =
3652 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
3653 ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
3654 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3655 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
3656 Argument = DAG.getLoad(
3657 PtrVT, dl, DAG.getEntryNode(), Argument,
3658 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3659 SDValue Chain = Argument.getValue(1);
3660
3661 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3662 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
3663
3664 // call __tls_get_addr.
3665 ArgListTy Args;
3666 ArgListEntry Entry;
3667 Entry.Node = Argument;
3668 Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
3669 Args.push_back(Entry);
3670
3671 // FIXME: is there useful debug info available here?
3672 TargetLowering::CallLoweringInfo CLI(DAG);
3673 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3674 CallingConv::C, Type::getInt32Ty(*DAG.getContext()),
3675 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args));
3676
3677 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3678 return CallResult.first;
3679 }
3680
3681 // Lower ISD::GlobalTLSAddress using the "initial exec" or
3682 // "local exec" model.
3683 SDValue
LowerToTLSExecModels(GlobalAddressSDNode * GA,SelectionDAG & DAG,TLSModel::Model model) const3684 ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
3685 SelectionDAG &DAG,
3686 TLSModel::Model model) const {
3687 const GlobalValue *GV = GA->getGlobal();
3688 SDLoc dl(GA);
3689 SDValue Offset;
3690 SDValue Chain = DAG.getEntryNode();
3691 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3692 // Get the Thread Pointer
3693 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3694
3695 if (model == TLSModel::InitialExec) {
3696 MachineFunction &MF = DAG.getMachineFunction();
3697 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3698 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3699 // Initial exec model.
3700 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3701 ARMConstantPoolValue *CPV =
3702 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
3703 ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
3704 true);
3705 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3706 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3707 Offset = DAG.getLoad(
3708 PtrVT, dl, Chain, Offset,
3709 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3710 Chain = Offset.getValue(1);
3711
3712 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3713 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
3714
3715 Offset = DAG.getLoad(
3716 PtrVT, dl, Chain, Offset,
3717 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3718 } else {
3719 // local exec model
3720 assert(model == TLSModel::LocalExec);
3721 ARMConstantPoolValue *CPV =
3722 ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF);
3723 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3724 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3725 Offset = DAG.getLoad(
3726 PtrVT, dl, Chain, Offset,
3727 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3728 }
3729
3730 // The address of the thread local variable is the add of the thread
3731 // pointer with the offset of the variable.
3732 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
3733 }
3734
3735 SDValue
LowerGlobalTLSAddress(SDValue Op,SelectionDAG & DAG) const3736 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3737 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
3738 if (DAG.getTarget().useEmulatedTLS())
3739 return LowerToTLSEmulatedModel(GA, DAG);
3740
3741 if (Subtarget->isTargetDarwin())
3742 return LowerGlobalTLSAddressDarwin(Op, DAG);
3743
3744 if (Subtarget->isTargetWindows())
3745 return LowerGlobalTLSAddressWindows(Op, DAG);
3746
3747 // TODO: implement the "local dynamic" model
3748 assert(Subtarget->isTargetELF() && "Only ELF implemented here");
3749 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal());
3750
3751 switch (model) {
3752 case TLSModel::GeneralDynamic:
3753 case TLSModel::LocalDynamic:
3754 return LowerToTLSGeneralDynamicModel(GA, DAG);
3755 case TLSModel::InitialExec:
3756 case TLSModel::LocalExec:
3757 return LowerToTLSExecModels(GA, DAG, model);
3758 }
3759 llvm_unreachable("bogus TLS model");
3760 }
3761
3762 /// Return true if all users of V are within function F, looking through
3763 /// ConstantExprs.
allUsersAreInFunction(const Value * V,const Function * F)3764 static bool allUsersAreInFunction(const Value *V, const Function *F) {
3765 SmallVector<const User*,4> Worklist(V->users());
3766 while (!Worklist.empty()) {
3767 auto *U = Worklist.pop_back_val();
3768 if (isa<ConstantExpr>(U)) {
3769 append_range(Worklist, U->users());
3770 continue;
3771 }
3772
3773 auto *I = dyn_cast<Instruction>(U);
3774 if (!I || I->getParent()->getParent() != F)
3775 return false;
3776 }
3777 return true;
3778 }
3779
promoteToConstantPool(const ARMTargetLowering * TLI,const GlobalValue * GV,SelectionDAG & DAG,EVT PtrVT,const SDLoc & dl)3780 static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
3781 const GlobalValue *GV, SelectionDAG &DAG,
3782 EVT PtrVT, const SDLoc &dl) {
3783 // If we're creating a pool entry for a constant global with unnamed address,
3784 // and the global is small enough, we can emit it inline into the constant pool
3785 // to save ourselves an indirection.
3786 //
3787 // This is a win if the constant is only used in one function (so it doesn't
3788 // need to be duplicated) or duplicating the constant wouldn't increase code
3789 // size (implying the constant is no larger than 4 bytes).
3790 const Function &F = DAG.getMachineFunction().getFunction();
3791
3792 // We rely on this decision to inline being idemopotent and unrelated to the
3793 // use-site. We know that if we inline a variable at one use site, we'll
3794 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
3795 // doesn't know about this optimization, so bail out if it's enabled else
3796 // we could decide to inline here (and thus never emit the GV) but require
3797 // the GV from fast-isel generated code.
3798 if (!EnableConstpoolPromotion ||
3799 DAG.getMachineFunction().getTarget().Options.EnableFastISel)
3800 return SDValue();
3801
3802 auto *GVar = dyn_cast<GlobalVariable>(GV);
3803 if (!GVar || !GVar->hasInitializer() ||
3804 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3805 !GVar->hasLocalLinkage())
3806 return SDValue();
3807
3808 // If we inline a value that contains relocations, we move the relocations
3809 // from .data to .text. This is not allowed in position-independent code.
3810 auto *Init = GVar->getInitializer();
3811 if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) &&
3812 Init->needsDynamicRelocation())
3813 return SDValue();
3814
3815 // The constant islands pass can only really deal with alignment requests
3816 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
3817 // any type wanting greater alignment requirements than 4 bytes. We also
3818 // can only promote constants that are multiples of 4 bytes in size or
3819 // are paddable to a multiple of 4. Currently we only try and pad constants
3820 // that are strings for simplicity.
3821 auto *CDAInit = dyn_cast<ConstantDataArray>(Init);
3822 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
3823 Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GVar);
3824 unsigned RequiredPadding = 4 - (Size % 4);
3825 bool PaddingPossible =
3826 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3827 if (!PaddingPossible || PrefAlign > 4 || Size > ConstpoolPromotionMaxSize ||
3828 Size == 0)
3829 return SDValue();
3830
3831 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3832 MachineFunction &MF = DAG.getMachineFunction();
3833 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3834
3835 // We can't bloat the constant pool too much, else the ConstantIslands pass
3836 // may fail to converge. If we haven't promoted this global yet (it may have
3837 // multiple uses), and promoting it would increase the constant pool size (Sz
3838 // > 4), ensure we have space to do so up to MaxTotal.
3839 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4)
3840 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
3841 ConstpoolPromotionMaxTotal)
3842 return SDValue();
3843
3844 // This is only valid if all users are in a single function; we can't clone
3845 // the constant in general. The LLVM IR unnamed_addr allows merging
3846 // constants, but not cloning them.
3847 //
3848 // We could potentially allow cloning if we could prove all uses of the
3849 // constant in the current function don't care about the address, like
3850 // printf format strings. But that isn't implemented for now.
3851 if (!allUsersAreInFunction(GVar, &F))
3852 return SDValue();
3853
3854 // We're going to inline this global. Pad it out if needed.
3855 if (RequiredPadding != 4) {
3856 StringRef S = CDAInit->getAsString();
3857
3858 SmallVector<uint8_t,16> V(S.size());
3859 std::copy(S.bytes_begin(), S.bytes_end(), V.begin());
3860 while (RequiredPadding--)
3861 V.push_back(0);
3862 Init = ConstantDataArray::get(*DAG.getContext(), V);
3863 }
3864
3865 auto CPVal = ARMConstantPoolConstant::Create(GVar, Init);
3866 SDValue CPAddr = DAG.getTargetConstantPool(CPVal, PtrVT, Align(4));
3867 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) {
3868 AFI->markGlobalAsPromotedToConstantPool(GVar);
3869 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() +
3870 PaddedSize - 4);
3871 }
3872 ++NumConstpoolPromoted;
3873 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3874 }
3875
isReadOnly(const GlobalValue * GV) const3876 bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const {
3877 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3878 if (!(GV = GA->getAliaseeObject()))
3879 return false;
3880 if (const auto *V = dyn_cast<GlobalVariable>(GV))
3881 return V->isConstant();
3882 return isa<Function>(GV);
3883 }
3884
LowerGlobalAddress(SDValue Op,SelectionDAG & DAG) const3885 SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3886 SelectionDAG &DAG) const {
3887 switch (Subtarget->getTargetTriple().getObjectFormat()) {
3888 default: llvm_unreachable("unknown object format");
3889 case Triple::COFF:
3890 return LowerGlobalAddressWindows(Op, DAG);
3891 case Triple::ELF:
3892 return LowerGlobalAddressELF(Op, DAG);
3893 case Triple::MachO:
3894 return LowerGlobalAddressDarwin(Op, DAG);
3895 }
3896 }
3897
LowerGlobalAddressELF(SDValue Op,SelectionDAG & DAG) const3898 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3899 SelectionDAG &DAG) const {
3900 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3901 SDLoc dl(Op);
3902 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3903 const TargetMachine &TM = getTargetMachine();
3904 bool IsRO = isReadOnly(GV);
3905
3906 // promoteToConstantPool only if not generating XO text section
3907 if (TM.shouldAssumeDSOLocal(*GV->getParent(), GV) && !Subtarget->genExecuteOnly())
3908 if (SDValue V = promoteToConstantPool(this, GV, DAG, PtrVT, dl))
3909 return V;
3910
3911 if (isPositionIndependent()) {
3912 bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV);
3913 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3914 UseGOT_PREL ? ARMII::MO_GOT : 0);
3915 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3916 if (UseGOT_PREL)
3917 Result =
3918 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
3919 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3920 return Result;
3921 } else if (Subtarget->isROPI() && IsRO) {
3922 // PC-relative.
3923 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
3924 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3925 return Result;
3926 } else if (Subtarget->isRWPI() && !IsRO) {
3927 // SB-relative.
3928 SDValue RelAddr;
3929 if (Subtarget->useMovt()) {
3930 ++NumMovwMovt;
3931 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
3932 RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G);
3933 } else { // use literal pool for address constant
3934 ARMConstantPoolValue *CPV =
3935 ARMConstantPoolConstant::Create(GV, ARMCP::SBREL);
3936 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3937 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3938 RelAddr = DAG.getLoad(
3939 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3940 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3941 }
3942 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
3943 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr);
3944 return Result;
3945 }
3946
3947 // If we have T2 ops, we can materialize the address directly via movt/movw
3948 // pair. This is always cheaper.
3949 if (Subtarget->useMovt()) {
3950 ++NumMovwMovt;
3951 // FIXME: Once remat is capable of dealing with instructions with register
3952 // operands, expand this into two nodes.
3953 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT,
3954 DAG.getTargetGlobalAddress(GV, dl, PtrVT));
3955 } else {
3956 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, Align(4));
3957 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3958 return DAG.getLoad(
3959 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3960 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3961 }
3962 }
3963
LowerGlobalAddressDarwin(SDValue Op,SelectionDAG & DAG) const3964 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3965 SelectionDAG &DAG) const {
3966 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3967 "ROPI/RWPI not currently supported for Darwin");
3968 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3969 SDLoc dl(Op);
3970 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3971
3972 if (Subtarget->useMovt())
3973 ++NumMovwMovt;
3974
3975 // FIXME: Once remat is capable of dealing with instructions with register
3976 // operands, expand this into multiple nodes
3977 unsigned Wrapper =
3978 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
3979
3980 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
3981 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
3982
3983 if (Subtarget->isGVIndirectSymbol(GV))
3984 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
3985 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3986 return Result;
3987 }
3988
LowerGlobalAddressWindows(SDValue Op,SelectionDAG & DAG) const3989 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3990 SelectionDAG &DAG) const {
3991 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported");
3992 assert(Subtarget->useMovt() &&
3993 "Windows on ARM expects to use movw/movt");
3994 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3995 "ROPI/RWPI not currently supported for Windows");
3996
3997 const TargetMachine &TM = getTargetMachine();
3998 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3999 ARMII::TOF TargetFlags = ARMII::MO_NO_FLAG;
4000 if (GV->hasDLLImportStorageClass())
4001 TargetFlags = ARMII::MO_DLLIMPORT;
4002 else if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
4003 TargetFlags = ARMII::MO_COFFSTUB;
4004 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4005 SDValue Result;
4006 SDLoc DL(Op);
4007
4008 ++NumMovwMovt;
4009
4010 // FIXME: Once remat is capable of dealing with instructions with register
4011 // operands, expand this into two nodes.
4012 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT,
4013 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*offset=*/0,
4014 TargetFlags));
4015 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
4016 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
4017 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4018 return Result;
4019 }
4020
4021 SDValue
LowerEH_SJLJ_SETJMP(SDValue Op,SelectionDAG & DAG) const4022 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
4023 SDLoc dl(Op);
4024 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
4025 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
4026 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
4027 Op.getOperand(1), Val);
4028 }
4029
4030 SDValue
LowerEH_SJLJ_LONGJMP(SDValue Op,SelectionDAG & DAG) const4031 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
4032 SDLoc dl(Op);
4033 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
4034 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
4035 }
4036
LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,SelectionDAG & DAG) const4037 SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
4038 SelectionDAG &DAG) const {
4039 SDLoc dl(Op);
4040 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
4041 Op.getOperand(0));
4042 }
4043
LowerINTRINSIC_VOID(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget) const4044 SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4045 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
4046 unsigned IntNo =
4047 cast<ConstantSDNode>(
4048 Op.getOperand(Op.getOperand(0).getValueType() == MVT::Other))
4049 ->getZExtValue();
4050 switch (IntNo) {
4051 default:
4052 return SDValue(); // Don't custom lower most intrinsics.
4053 case Intrinsic::arm_gnu_eabi_mcount: {
4054 MachineFunction &MF = DAG.getMachineFunction();
4055 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4056 SDLoc dl(Op);
4057 SDValue Chain = Op.getOperand(0);
4058 // call "\01__gnu_mcount_nc"
4059 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
4060 const uint32_t *Mask =
4061 ARI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
4062 assert(Mask && "Missing call preserved mask for calling convention");
4063 // Mark LR an implicit live-in.
4064 Register Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
4065 SDValue ReturnAddress =
4066 DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, PtrVT);
4067 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4068 SDValue Callee =
4069 DAG.getTargetExternalSymbol("\01__gnu_mcount_nc", PtrVT, 0);
4070 SDValue RegisterMask = DAG.getRegisterMask(Mask);
4071 if (Subtarget->isThumb())
4072 return SDValue(
4073 DAG.getMachineNode(
4074 ARM::tBL_PUSHLR, dl, ResultTys,
4075 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4076 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4077 0);
4078 return SDValue(
4079 DAG.getMachineNode(ARM::BL_PUSHLR, dl, ResultTys,
4080 {ReturnAddress, Callee, RegisterMask, Chain}),
4081 0);
4082 }
4083 }
4084 }
4085
4086 SDValue
LowerINTRINSIC_WO_CHAIN(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget) const4087 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
4088 const ARMSubtarget *Subtarget) const {
4089 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4090 SDLoc dl(Op);
4091 switch (IntNo) {
4092 default: return SDValue(); // Don't custom lower most intrinsics.
4093 case Intrinsic::thread_pointer: {
4094 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4095 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
4096 }
4097 case Intrinsic::arm_cls: {
4098 const SDValue &Operand = Op.getOperand(1);
4099 const EVT VTy = Op.getValueType();
4100 SDValue SRA =
4101 DAG.getNode(ISD::SRA, dl, VTy, Operand, DAG.getConstant(31, dl, VTy));
4102 SDValue XOR = DAG.getNode(ISD::XOR, dl, VTy, SRA, Operand);
4103 SDValue SHL =
4104 DAG.getNode(ISD::SHL, dl, VTy, XOR, DAG.getConstant(1, dl, VTy));
4105 SDValue OR =
4106 DAG.getNode(ISD::OR, dl, VTy, SHL, DAG.getConstant(1, dl, VTy));
4107 SDValue Result = DAG.getNode(ISD::CTLZ, dl, VTy, OR);
4108 return Result;
4109 }
4110 case Intrinsic::arm_cls64: {
4111 // cls(x) = if cls(hi(x)) != 31 then cls(hi(x))
4112 // else 31 + clz(if hi(x) == 0 then lo(x) else not(lo(x)))
4113 const SDValue &Operand = Op.getOperand(1);
4114 const EVT VTy = Op.getValueType();
4115
4116 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
4117 DAG.getConstant(1, dl, VTy));
4118 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
4119 DAG.getConstant(0, dl, VTy));
4120 SDValue Constant0 = DAG.getConstant(0, dl, VTy);
4121 SDValue Constant1 = DAG.getConstant(1, dl, VTy);
4122 SDValue Constant31 = DAG.getConstant(31, dl, VTy);
4123 SDValue SRAHi = DAG.getNode(ISD::SRA, dl, VTy, Hi, Constant31);
4124 SDValue XORHi = DAG.getNode(ISD::XOR, dl, VTy, SRAHi, Hi);
4125 SDValue SHLHi = DAG.getNode(ISD::SHL, dl, VTy, XORHi, Constant1);
4126 SDValue ORHi = DAG.getNode(ISD::OR, dl, VTy, SHLHi, Constant1);
4127 SDValue CLSHi = DAG.getNode(ISD::CTLZ, dl, VTy, ORHi);
4128 SDValue CheckLo =
4129 DAG.getSetCC(dl, MVT::i1, CLSHi, Constant31, ISD::CondCode::SETEQ);
4130 SDValue HiIsZero =
4131 DAG.getSetCC(dl, MVT::i1, Hi, Constant0, ISD::CondCode::SETEQ);
4132 SDValue AdjustedLo =
4133 DAG.getSelect(dl, VTy, HiIsZero, Lo, DAG.getNOT(dl, Lo, VTy));
4134 SDValue CLZAdjustedLo = DAG.getNode(ISD::CTLZ, dl, VTy, AdjustedLo);
4135 SDValue Result =
4136 DAG.getSelect(dl, VTy, CheckLo,
4137 DAG.getNode(ISD::ADD, dl, VTy, CLZAdjustedLo, Constant31), CLSHi);
4138 return Result;
4139 }
4140 case Intrinsic::eh_sjlj_lsda: {
4141 MachineFunction &MF = DAG.getMachineFunction();
4142 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4143 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
4144 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4145 SDValue CPAddr;
4146 bool IsPositionIndependent = isPositionIndependent();
4147 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4148 ARMConstantPoolValue *CPV =
4149 ARMConstantPoolConstant::Create(&MF.getFunction(), ARMPCLabelIndex,
4150 ARMCP::CPLSDA, PCAdj);
4151 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
4152 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
4153 SDValue Result = DAG.getLoad(
4154 PtrVT, dl, DAG.getEntryNode(), CPAddr,
4155 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4156
4157 if (IsPositionIndependent) {
4158 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
4159 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
4160 }
4161 return Result;
4162 }
4163 case Intrinsic::arm_neon_vabs:
4164 return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(),
4165 Op.getOperand(1));
4166 case Intrinsic::arm_neon_vmulls:
4167 case Intrinsic::arm_neon_vmullu: {
4168 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4169 ? ARMISD::VMULLs : ARMISD::VMULLu;
4170 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4171 Op.getOperand(1), Op.getOperand(2));
4172 }
4173 case Intrinsic::arm_neon_vminnm:
4174 case Intrinsic::arm_neon_vmaxnm: {
4175 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4176 ? ISD::FMINNUM : ISD::FMAXNUM;
4177 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4178 Op.getOperand(1), Op.getOperand(2));
4179 }
4180 case Intrinsic::arm_neon_vminu:
4181 case Intrinsic::arm_neon_vmaxu: {
4182 if (Op.getValueType().isFloatingPoint())
4183 return SDValue();
4184 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4185 ? ISD::UMIN : ISD::UMAX;
4186 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4187 Op.getOperand(1), Op.getOperand(2));
4188 }
4189 case Intrinsic::arm_neon_vmins:
4190 case Intrinsic::arm_neon_vmaxs: {
4191 // v{min,max}s is overloaded between signed integers and floats.
4192 if (!Op.getValueType().isFloatingPoint()) {
4193 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4194 ? ISD::SMIN : ISD::SMAX;
4195 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4196 Op.getOperand(1), Op.getOperand(2));
4197 }
4198 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4199 ? ISD::FMINIMUM : ISD::FMAXIMUM;
4200 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4201 Op.getOperand(1), Op.getOperand(2));
4202 }
4203 case Intrinsic::arm_neon_vtbl1:
4204 return DAG.getNode(ARMISD::VTBL1, SDLoc(Op), Op.getValueType(),
4205 Op.getOperand(1), Op.getOperand(2));
4206 case Intrinsic::arm_neon_vtbl2:
4207 return DAG.getNode(ARMISD::VTBL2, SDLoc(Op), Op.getValueType(),
4208 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4209 case Intrinsic::arm_mve_pred_i2v:
4210 case Intrinsic::arm_mve_pred_v2i:
4211 return DAG.getNode(ARMISD::PREDICATE_CAST, SDLoc(Op), Op.getValueType(),
4212 Op.getOperand(1));
4213 case Intrinsic::arm_mve_vreinterpretq:
4214 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(Op), Op.getValueType(),
4215 Op.getOperand(1));
4216 case Intrinsic::arm_mve_lsll:
4217 return DAG.getNode(ARMISD::LSLL, SDLoc(Op), Op->getVTList(),
4218 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4219 case Intrinsic::arm_mve_asrl:
4220 return DAG.getNode(ARMISD::ASRL, SDLoc(Op), Op->getVTList(),
4221 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4222 }
4223 }
4224
LowerATOMIC_FENCE(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)4225 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
4226 const ARMSubtarget *Subtarget) {
4227 SDLoc dl(Op);
4228 ConstantSDNode *SSIDNode = cast<ConstantSDNode>(Op.getOperand(2));
4229 auto SSID = static_cast<SyncScope::ID>(SSIDNode->getZExtValue());
4230 if (SSID == SyncScope::SingleThread)
4231 return Op;
4232
4233 if (!Subtarget->hasDataBarrier()) {
4234 // Some ARMv6 cpus can support data barriers with an mcr instruction.
4235 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
4236 // here.
4237 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4238 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4239 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
4240 DAG.getConstant(0, dl, MVT::i32));
4241 }
4242
4243 ConstantSDNode *OrdN = cast<ConstantSDNode>(Op.getOperand(1));
4244 AtomicOrdering Ord = static_cast<AtomicOrdering>(OrdN->getZExtValue());
4245 ARM_MB::MemBOpt Domain = ARM_MB::ISH;
4246 if (Subtarget->isMClass()) {
4247 // Only a full system barrier exists in the M-class architectures.
4248 Domain = ARM_MB::SY;
4249 } else if (Subtarget->preferISHSTBarriers() &&
4250 Ord == AtomicOrdering::Release) {
4251 // Swift happens to implement ISHST barriers in a way that's compatible with
4252 // Release semantics but weaker than ISH so we'd be fools not to use
4253 // it. Beware: other processors probably don't!
4254 Domain = ARM_MB::ISHST;
4255 }
4256
4257 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
4258 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4259 DAG.getConstant(Domain, dl, MVT::i32));
4260 }
4261
LowerPREFETCH(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)4262 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
4263 const ARMSubtarget *Subtarget) {
4264 // ARM pre v5TE and Thumb1 does not have preload instructions.
4265 if (!(Subtarget->isThumb2() ||
4266 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
4267 // Just preserve the chain.
4268 return Op.getOperand(0);
4269
4270 SDLoc dl(Op);
4271 unsigned isRead = ~cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() & 1;
4272 if (!isRead &&
4273 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4274 // ARMv7 with MP extension has PLDW.
4275 return Op.getOperand(0);
4276
4277 unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
4278 if (Subtarget->isThumb()) {
4279 // Invert the bits.
4280 isRead = ~isRead & 1;
4281 isData = ~isData & 1;
4282 }
4283
4284 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
4285 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
4286 DAG.getConstant(isData, dl, MVT::i32));
4287 }
4288
LowerVASTART(SDValue Op,SelectionDAG & DAG)4289 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
4290 MachineFunction &MF = DAG.getMachineFunction();
4291 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
4292
4293 // vastart just stores the address of the VarArgsFrameIndex slot into the
4294 // memory location argument.
4295 SDLoc dl(Op);
4296 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
4297 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4298 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4299 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
4300 MachinePointerInfo(SV));
4301 }
4302
GetF64FormalArgument(CCValAssign & VA,CCValAssign & NextVA,SDValue & Root,SelectionDAG & DAG,const SDLoc & dl) const4303 SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
4304 CCValAssign &NextVA,
4305 SDValue &Root,
4306 SelectionDAG &DAG,
4307 const SDLoc &dl) const {
4308 MachineFunction &MF = DAG.getMachineFunction();
4309 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4310
4311 const TargetRegisterClass *RC;
4312 if (AFI->isThumb1OnlyFunction())
4313 RC = &ARM::tGPRRegClass;
4314 else
4315 RC = &ARM::GPRRegClass;
4316
4317 // Transform the arguments stored in physical registers into virtual ones.
4318 Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
4319 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4320
4321 SDValue ArgValue2;
4322 if (NextVA.isMemLoc()) {
4323 MachineFrameInfo &MFI = MF.getFrameInfo();
4324 int FI = MFI.CreateFixedObject(4, NextVA.getLocMemOffset(), true);
4325
4326 // Create load node to retrieve arguments from the stack.
4327 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4328 ArgValue2 = DAG.getLoad(
4329 MVT::i32, dl, Root, FIN,
4330 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
4331 } else {
4332 Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
4333 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4334 }
4335 if (!Subtarget->isLittle())
4336 std::swap (ArgValue, ArgValue2);
4337 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
4338 }
4339
4340 // The remaining GPRs hold either the beginning of variable-argument
4341 // data, or the beginning of an aggregate passed by value (usually
4342 // byval). Either way, we allocate stack slots adjacent to the data
4343 // provided by our caller, and store the unallocated registers there.
4344 // If this is a variadic function, the va_list pointer will begin with
4345 // these values; otherwise, this reassembles a (byval) structure that
4346 // was split between registers and memory.
4347 // Return: The frame index registers were stored into.
StoreByValRegs(CCState & CCInfo,SelectionDAG & DAG,const SDLoc & dl,SDValue & Chain,const Value * OrigArg,unsigned InRegsParamRecordIdx,int ArgOffset,unsigned ArgSize) const4348 int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
4349 const SDLoc &dl, SDValue &Chain,
4350 const Value *OrigArg,
4351 unsigned InRegsParamRecordIdx,
4352 int ArgOffset, unsigned ArgSize) const {
4353 // Currently, two use-cases possible:
4354 // Case #1. Non-var-args function, and we meet first byval parameter.
4355 // Setup first unallocated register as first byval register;
4356 // eat all remained registers
4357 // (these two actions are performed by HandleByVal method).
4358 // Then, here, we initialize stack frame with
4359 // "store-reg" instructions.
4360 // Case #2. Var-args function, that doesn't contain byval parameters.
4361 // The same: eat all remained unallocated registers,
4362 // initialize stack frame.
4363
4364 MachineFunction &MF = DAG.getMachineFunction();
4365 MachineFrameInfo &MFI = MF.getFrameInfo();
4366 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4367 unsigned RBegin, REnd;
4368 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
4369 CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd);
4370 } else {
4371 unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
4372 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
4373 REnd = ARM::R4;
4374 }
4375
4376 if (REnd != RBegin)
4377 ArgOffset = -4 * (ARM::R4 - RBegin);
4378
4379 auto PtrVT = getPointerTy(DAG.getDataLayout());
4380 int FrameIndex = MFI.CreateFixedObject(ArgSize, ArgOffset, false);
4381 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
4382
4383 SmallVector<SDValue, 4> MemOps;
4384 const TargetRegisterClass *RC =
4385 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
4386
4387 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
4388 Register VReg = MF.addLiveIn(Reg, RC);
4389 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
4390 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
4391 MachinePointerInfo(OrigArg, 4 * i));
4392 MemOps.push_back(Store);
4393 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
4394 }
4395
4396 if (!MemOps.empty())
4397 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
4398 return FrameIndex;
4399 }
4400
4401 // Setup stack frame, the va_list pointer will start from.
VarArgStyleRegisters(CCState & CCInfo,SelectionDAG & DAG,const SDLoc & dl,SDValue & Chain,unsigned ArgOffset,unsigned TotalArgRegsSaveSize,bool ForceMutable) const4402 void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
4403 const SDLoc &dl, SDValue &Chain,
4404 unsigned ArgOffset,
4405 unsigned TotalArgRegsSaveSize,
4406 bool ForceMutable) const {
4407 MachineFunction &MF = DAG.getMachineFunction();
4408 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4409
4410 // Try to store any remaining integer argument regs
4411 // to their spots on the stack so that they may be loaded by dereferencing
4412 // the result of va_next.
4413 // If there is no regs to be stored, just point address after last
4414 // argument passed via stack.
4415 int FrameIndex = StoreByValRegs(CCInfo, DAG, dl, Chain, nullptr,
4416 CCInfo.getInRegsParamsCount(),
4417 CCInfo.getNextStackOffset(),
4418 std::max(4U, TotalArgRegsSaveSize));
4419 AFI->setVarArgsFrameIndex(FrameIndex);
4420 }
4421
splitValueIntoRegisterParts(SelectionDAG & DAG,const SDLoc & DL,SDValue Val,SDValue * Parts,unsigned NumParts,MVT PartVT,std::optional<CallingConv::ID> CC) const4422 bool ARMTargetLowering::splitValueIntoRegisterParts(
4423 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4424 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4425 bool IsABIRegCopy = CC.has_value();
4426 EVT ValueVT = Val.getValueType();
4427 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
4428 PartVT == MVT::f32) {
4429 unsigned ValueBits = ValueVT.getSizeInBits();
4430 unsigned PartBits = PartVT.getSizeInBits();
4431 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val);
4432 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val);
4433 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
4434 Parts[0] = Val;
4435 return true;
4436 }
4437 return false;
4438 }
4439
joinRegisterPartsIntoValue(SelectionDAG & DAG,const SDLoc & DL,const SDValue * Parts,unsigned NumParts,MVT PartVT,EVT ValueVT,std::optional<CallingConv::ID> CC) const4440 SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4441 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4442 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
4443 bool IsABIRegCopy = CC.has_value();
4444 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
4445 PartVT == MVT::f32) {
4446 unsigned ValueBits = ValueVT.getSizeInBits();
4447 unsigned PartBits = PartVT.getSizeInBits();
4448 SDValue Val = Parts[0];
4449
4450 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val);
4451 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val);
4452 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
4453 return Val;
4454 }
4455 return SDValue();
4456 }
4457
LowerFormalArguments(SDValue Chain,CallingConv::ID CallConv,bool isVarArg,const SmallVectorImpl<ISD::InputArg> & Ins,const SDLoc & dl,SelectionDAG & DAG,SmallVectorImpl<SDValue> & InVals) const4458 SDValue ARMTargetLowering::LowerFormalArguments(
4459 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4460 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4461 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4462 MachineFunction &MF = DAG.getMachineFunction();
4463 MachineFrameInfo &MFI = MF.getFrameInfo();
4464
4465 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4466
4467 // Assign locations to all of the incoming arguments.
4468 SmallVector<CCValAssign, 16> ArgLocs;
4469 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4470 *DAG.getContext());
4471 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg));
4472
4473 SmallVector<SDValue, 16> ArgValues;
4474 SDValue ArgValue;
4475 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
4476 unsigned CurArgIdx = 0;
4477
4478 // Initially ArgRegsSaveSize is zero.
4479 // Then we increase this value each time we meet byval parameter.
4480 // We also increase this value in case of varargs function.
4481 AFI->setArgRegsSaveSize(0);
4482
4483 // Calculate the amount of stack space that we need to allocate to store
4484 // byval and variadic arguments that are passed in registers.
4485 // We need to know this before we allocate the first byval or variadic
4486 // argument, as they will be allocated a stack slot below the CFA (Canonical
4487 // Frame Address, the stack pointer at entry to the function).
4488 unsigned ArgRegBegin = ARM::R4;
4489 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4490 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
4491 break;
4492
4493 CCValAssign &VA = ArgLocs[i];
4494 unsigned Index = VA.getValNo();
4495 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
4496 if (!Flags.isByVal())
4497 continue;
4498
4499 assert(VA.isMemLoc() && "unexpected byval pointer in reg");
4500 unsigned RBegin, REnd;
4501 CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd);
4502 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4503
4504 CCInfo.nextInRegsParam();
4505 }
4506 CCInfo.rewindByValRegsInfo();
4507
4508 int lastInsIndex = -1;
4509 if (isVarArg && MFI.hasVAStart()) {
4510 unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
4511 if (RegIdx != std::size(GPRArgRegs))
4512 ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]);
4513 }
4514
4515 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4516 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
4517 auto PtrVT = getPointerTy(DAG.getDataLayout());
4518
4519 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4520 CCValAssign &VA = ArgLocs[i];
4521 if (Ins[VA.getValNo()].isOrigArg()) {
4522 std::advance(CurOrigArg,
4523 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
4524 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
4525 }
4526 // Arguments stored in registers.
4527 if (VA.isRegLoc()) {
4528 EVT RegVT = VA.getLocVT();
4529
4530 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
4531 // f64 and vector types are split up into multiple registers or
4532 // combinations of registers and stack slots.
4533 SDValue ArgValue1 =
4534 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4535 VA = ArgLocs[++i]; // skip ahead to next loc
4536 SDValue ArgValue2;
4537 if (VA.isMemLoc()) {
4538 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), true);
4539 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4540 ArgValue2 = DAG.getLoad(
4541 MVT::f64, dl, Chain, FIN,
4542 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
4543 } else {
4544 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4545 }
4546 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
4547 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4548 ArgValue1, DAG.getIntPtrConstant(0, dl));
4549 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4550 ArgValue2, DAG.getIntPtrConstant(1, dl));
4551 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
4552 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4553 } else {
4554 const TargetRegisterClass *RC;
4555
4556 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4557 RC = &ARM::HPRRegClass;
4558 else if (RegVT == MVT::f32)
4559 RC = &ARM::SPRRegClass;
4560 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4561 RegVT == MVT::v4bf16)
4562 RC = &ARM::DPRRegClass;
4563 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4564 RegVT == MVT::v8bf16)
4565 RC = &ARM::QPRRegClass;
4566 else if (RegVT == MVT::i32)
4567 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
4568 : &ARM::GPRRegClass;
4569 else
4570 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
4571
4572 // Transform the arguments in physical registers into virtual ones.
4573 Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
4574 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
4575
4576 // If this value is passed in r0 and has the returned attribute (e.g.
4577 // C++ 'structors), record this fact for later use.
4578 if (VA.getLocReg() == ARM::R0 && Ins[VA.getValNo()].Flags.isReturned()) {
4579 AFI->setPreservesR0();
4580 }
4581 }
4582
4583 // If this is an 8 or 16-bit value, it is really passed promoted
4584 // to 32 bits. Insert an assert[sz]ext to capture this, then
4585 // truncate to the right size.
4586 switch (VA.getLocInfo()) {
4587 default: llvm_unreachable("Unknown loc info!");
4588 case CCValAssign::Full: break;
4589 case CCValAssign::BCvt:
4590 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
4591 break;
4592 case CCValAssign::SExt:
4593 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
4594 DAG.getValueType(VA.getValVT()));
4595 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
4596 break;
4597 case CCValAssign::ZExt:
4598 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
4599 DAG.getValueType(VA.getValVT()));
4600 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
4601 break;
4602 }
4603
4604 // f16 arguments have their size extended to 4 bytes and passed as if they
4605 // had been copied to the LSBs of a 32-bit register.
4606 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
4607 if (VA.needsCustom() &&
4608 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
4609 ArgValue = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), ArgValue);
4610
4611 InVals.push_back(ArgValue);
4612 } else { // VA.isRegLoc()
4613 // Only arguments passed on the stack should make it here.
4614 assert(VA.isMemLoc());
4615 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
4616
4617 int index = VA.getValNo();
4618
4619 // Some Ins[] entries become multiple ArgLoc[] entries.
4620 // Process them only once.
4621 if (index != lastInsIndex)
4622 {
4623 ISD::ArgFlagsTy Flags = Ins[index].Flags;
4624 // FIXME: For now, all byval parameter objects are marked mutable.
4625 // This can be changed with more analysis.
4626 // In case of tail call optimization mark all arguments mutable.
4627 // Since they could be overwritten by lowering of arguments in case of
4628 // a tail call.
4629 if (Flags.isByVal()) {
4630 assert(Ins[index].isOrigArg() &&
4631 "Byval arguments cannot be implicit");
4632 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
4633
4634 int FrameIndex = StoreByValRegs(
4635 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4636 VA.getLocMemOffset(), Flags.getByValSize());
4637 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
4638 CCInfo.nextInRegsParam();
4639 } else {
4640 unsigned FIOffset = VA.getLocMemOffset();
4641 int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits()/8,
4642 FIOffset, true);
4643
4644 // Create load nodes to retrieve arguments from the stack.
4645 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4646 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
4647 MachinePointerInfo::getFixedStack(
4648 DAG.getMachineFunction(), FI)));
4649 }
4650 lastInsIndex = index;
4651 }
4652 }
4653 }
4654
4655 // varargs
4656 if (isVarArg && MFI.hasVAStart()) {
4657 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.getNextStackOffset(),
4658 TotalArgRegsSaveSize);
4659 if (AFI->isCmseNSEntryFunction()) {
4660 DiagnosticInfoUnsupported Diag(
4661 DAG.getMachineFunction().getFunction(),
4662 "secure entry function must not be variadic", dl.getDebugLoc());
4663 DAG.getContext()->diagnose(Diag);
4664 }
4665 }
4666
4667 unsigned StackArgSize = CCInfo.getNextStackOffset();
4668 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4669 if (canGuaranteeTCO(CallConv, TailCallOpt)) {
4670 // The only way to guarantee a tail call is if the callee restores its
4671 // argument area, but it must also keep the stack aligned when doing so.
4672 const DataLayout &DL = DAG.getDataLayout();
4673 StackArgSize = alignTo(StackArgSize, DL.getStackAlignment());
4674
4675 AFI->setArgumentStackToRestore(StackArgSize);
4676 }
4677 AFI->setArgumentStackSize(StackArgSize);
4678
4679 if (CCInfo.getNextStackOffset() > 0 && AFI->isCmseNSEntryFunction()) {
4680 DiagnosticInfoUnsupported Diag(
4681 DAG.getMachineFunction().getFunction(),
4682 "secure entry function requires arguments on stack", dl.getDebugLoc());
4683 DAG.getContext()->diagnose(Diag);
4684 }
4685
4686 return Chain;
4687 }
4688
4689 /// isFloatingPointZero - Return true if this is +0.0.
isFloatingPointZero(SDValue Op)4690 static bool isFloatingPointZero(SDValue Op) {
4691 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
4692 return CFP->getValueAPF().isPosZero();
4693 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
4694 // Maybe this has already been legalized into the constant pool?
4695 if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
4696 SDValue WrapperOp = Op.getOperand(1).getOperand(0);
4697 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp))
4698 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4699 return CFP->getValueAPF().isPosZero();
4700 }
4701 } else if (Op->getOpcode() == ISD::BITCAST &&
4702 Op->getValueType(0) == MVT::f64) {
4703 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
4704 // created by LowerConstantFP().
4705 SDValue BitcastOp = Op->getOperand(0);
4706 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
4707 isNullConstant(BitcastOp->getOperand(0)))
4708 return true;
4709 }
4710 return false;
4711 }
4712
4713 /// Returns appropriate ARM CMP (cmp) and corresponding condition code for
4714 /// the given operands.
getARMCmp(SDValue LHS,SDValue RHS,ISD::CondCode CC,SDValue & ARMcc,SelectionDAG & DAG,const SDLoc & dl) const4715 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4716 SDValue &ARMcc, SelectionDAG &DAG,
4717 const SDLoc &dl) const {
4718 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
4719 unsigned C = RHSC->getZExtValue();
4720 if (!isLegalICmpImmediate((int32_t)C)) {
4721 // Constant does not fit, try adjusting it by one.
4722 switch (CC) {
4723 default: break;
4724 case ISD::SETLT:
4725 case ISD::SETGE:
4726 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) {
4727 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
4728 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4729 }
4730 break;
4731 case ISD::SETULT:
4732 case ISD::SETUGE:
4733 if (C != 0 && isLegalICmpImmediate(C-1)) {
4734 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
4735 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4736 }
4737 break;
4738 case ISD::SETLE:
4739 case ISD::SETGT:
4740 if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) {
4741 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
4742 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4743 }
4744 break;
4745 case ISD::SETULE:
4746 case ISD::SETUGT:
4747 if (C != 0xffffffff && isLegalICmpImmediate(C+1)) {
4748 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4749 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4750 }
4751 break;
4752 }
4753 }
4754 } else if ((ARM_AM::getShiftOpcForNode(LHS.getOpcode()) != ARM_AM::no_shift) &&
4755 (ARM_AM::getShiftOpcForNode(RHS.getOpcode()) == ARM_AM::no_shift)) {
4756 // In ARM and Thumb-2, the compare instructions can shift their second
4757 // operand.
4758 CC = ISD::getSetCCSwappedOperands(CC);
4759 std::swap(LHS, RHS);
4760 }
4761
4762 // Thumb1 has very limited immediate modes, so turning an "and" into a
4763 // shift can save multiple instructions.
4764 //
4765 // If we have (x & C1), and C1 is an appropriate mask, we can transform it
4766 // into "((x << n) >> n)". But that isn't necessarily profitable on its
4767 // own. If it's the operand to an unsigned comparison with an immediate,
4768 // we can eliminate one of the shifts: we transform
4769 // "((x << n) >> n) == C2" to "(x << n) == (C2 << n)".
4770 //
4771 // We avoid transforming cases which aren't profitable due to encoding
4772 // details:
4773 //
4774 // 1. C2 fits into the immediate field of a cmp, and the transformed version
4775 // would not; in that case, we're essentially trading one immediate load for
4776 // another.
4777 // 2. C1 is 255 or 65535, so we can use uxtb or uxth.
4778 // 3. C2 is zero; we have other code for this special case.
4779 //
4780 // FIXME: Figure out profitability for Thumb2; we usually can't save an
4781 // instruction, since the AND is always one instruction anyway, but we could
4782 // use narrow instructions in some cases.
4783 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND &&
4784 LHS->hasOneUse() && isa<ConstantSDNode>(LHS.getOperand(1)) &&
4785 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4786 !isSignedIntSetCC(CC)) {
4787 unsigned Mask = cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue();
4788 auto *RHSC = cast<ConstantSDNode>(RHS.getNode());
4789 uint64_t RHSV = RHSC->getZExtValue();
4790 if (isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4791 unsigned ShiftBits = countLeadingZeros(Mask);
4792 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4793 SDValue ShiftAmt = DAG.getConstant(ShiftBits, dl, MVT::i32);
4794 LHS = DAG.getNode(ISD::SHL, dl, MVT::i32, LHS.getOperand(0), ShiftAmt);
4795 RHS = DAG.getConstant(RHSV << ShiftBits, dl, MVT::i32);
4796 }
4797 }
4798 }
4799
4800 // The specific comparison "(x<<c) > 0x80000000U" can be optimized to a
4801 // single "lsls x, c+1". The shift sets the "C" and "Z" flags the same
4802 // way a cmp would.
4803 // FIXME: Add support for ARM/Thumb2; this would need isel patterns, and
4804 // some tweaks to the heuristics for the previous and->shift transform.
4805 // FIXME: Optimize cases where the LHS isn't a shift.
4806 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL &&
4807 isa<ConstantSDNode>(RHS) &&
4808 cast<ConstantSDNode>(RHS)->getZExtValue() == 0x80000000U &&
4809 CC == ISD::SETUGT && isa<ConstantSDNode>(LHS.getOperand(1)) &&
4810 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() < 31) {
4811 unsigned ShiftAmt =
4812 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() + 1;
4813 SDValue Shift = DAG.getNode(ARMISD::LSLS, dl,
4814 DAG.getVTList(MVT::i32, MVT::i32),
4815 LHS.getOperand(0),
4816 DAG.getConstant(ShiftAmt, dl, MVT::i32));
4817 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
4818 Shift.getValue(1), SDValue());
4819 ARMcc = DAG.getConstant(ARMCC::HI, dl, MVT::i32);
4820 return Chain.getValue(1);
4821 }
4822
4823 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
4824
4825 // If the RHS is a constant zero then the V (overflow) flag will never be
4826 // set. This can allow us to simplify GE to PL or LT to MI, which can be
4827 // simpler for other passes (like the peephole optimiser) to deal with.
4828 if (isNullConstant(RHS)) {
4829 switch (CondCode) {
4830 default: break;
4831 case ARMCC::GE:
4832 CondCode = ARMCC::PL;
4833 break;
4834 case ARMCC::LT:
4835 CondCode = ARMCC::MI;
4836 break;
4837 }
4838 }
4839
4840 ARMISD::NodeType CompareType;
4841 switch (CondCode) {
4842 default:
4843 CompareType = ARMISD::CMP;
4844 break;
4845 case ARMCC::EQ:
4846 case ARMCC::NE:
4847 // Uses only Z Flag
4848 CompareType = ARMISD::CMPZ;
4849 break;
4850 }
4851 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
4852 return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4853 }
4854
4855 /// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
getVFPCmp(SDValue LHS,SDValue RHS,SelectionDAG & DAG,const SDLoc & dl,bool Signaling) const4856 SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4857 SelectionDAG &DAG, const SDLoc &dl,
4858 bool Signaling) const {
4859 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64);
4860 SDValue Cmp;
4861 if (!isFloatingPointZero(RHS))
4862 Cmp = DAG.getNode(Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP,
4863 dl, MVT::Glue, LHS, RHS);
4864 else
4865 Cmp = DAG.getNode(Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0,
4866 dl, MVT::Glue, LHS);
4867 return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Glue, Cmp);
4868 }
4869
4870 /// duplicateCmp - Glue values can have only one use, so this function
4871 /// duplicates a comparison node.
4872 SDValue
duplicateCmp(SDValue Cmp,SelectionDAG & DAG) const4873 ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const {
4874 unsigned Opc = Cmp.getOpcode();
4875 SDLoc DL(Cmp);
4876 if (Opc == ARMISD::CMP || Opc == ARMISD::CMPZ)
4877 return DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
4878
4879 assert(Opc == ARMISD::FMSTAT && "unexpected comparison operation");
4880 Cmp = Cmp.getOperand(0);
4881 Opc = Cmp.getOpcode();
4882 if (Opc == ARMISD::CMPFP)
4883 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
4884 else {
4885 assert(Opc == ARMISD::CMPFPw0 && "unexpected operand of FMSTAT");
4886 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0));
4887 }
4888 return DAG.getNode(ARMISD::FMSTAT, DL, MVT::Glue, Cmp);
4889 }
4890
4891 // This function returns three things: the arithmetic computation itself
4892 // (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The
4893 // comparison and the condition code define the case in which the arithmetic
4894 // computation *does not* overflow.
4895 std::pair<SDValue, SDValue>
getARMXALUOOp(SDValue Op,SelectionDAG & DAG,SDValue & ARMcc) const4896 ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4897 SDValue &ARMcc) const {
4898 assert(Op.getValueType() == MVT::i32 && "Unsupported value type");
4899
4900 SDValue Value, OverflowCmp;
4901 SDValue LHS = Op.getOperand(0);
4902 SDValue RHS = Op.getOperand(1);
4903 SDLoc dl(Op);
4904
4905 // FIXME: We are currently always generating CMPs because we don't support
4906 // generating CMN through the backend. This is not as good as the natural
4907 // CMP case because it causes a register dependency and cannot be folded
4908 // later.
4909
4910 switch (Op.getOpcode()) {
4911 default:
4912 llvm_unreachable("Unknown overflow instruction!");
4913 case ISD::SADDO:
4914 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
4915 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
4916 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
4917 break;
4918 case ISD::UADDO:
4919 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
4920 // We use ADDC here to correspond to its use in LowerUnsignedALUO.
4921 // We do not use it in the USUBO case as Value may not be used.
4922 Value = DAG.getNode(ARMISD::ADDC, dl,
4923 DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS)
4924 .getValue(0);
4925 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
4926 break;
4927 case ISD::SSUBO:
4928 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
4929 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
4930 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
4931 break;
4932 case ISD::USUBO:
4933 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
4934 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
4935 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
4936 break;
4937 case ISD::UMULO:
4938 // We generate a UMUL_LOHI and then check if the high word is 0.
4939 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
4940 Value = DAG.getNode(ISD::UMUL_LOHI, dl,
4941 DAG.getVTList(Op.getValueType(), Op.getValueType()),
4942 LHS, RHS);
4943 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1),
4944 DAG.getConstant(0, dl, MVT::i32));
4945 Value = Value.getValue(0); // We only want the low 32 bits for the result.
4946 break;
4947 case ISD::SMULO:
4948 // We generate a SMUL_LOHI and then check if all the bits of the high word
4949 // are the same as the sign bit of the low word.
4950 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
4951 Value = DAG.getNode(ISD::SMUL_LOHI, dl,
4952 DAG.getVTList(Op.getValueType(), Op.getValueType()),
4953 LHS, RHS);
4954 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1),
4955 DAG.getNode(ISD::SRA, dl, Op.getValueType(),
4956 Value.getValue(0),
4957 DAG.getConstant(31, dl, MVT::i32)));
4958 Value = Value.getValue(0); // We only want the low 32 bits for the result.
4959 break;
4960 } // switch (...)
4961
4962 return std::make_pair(Value, OverflowCmp);
4963 }
4964
4965 SDValue
LowerSignedALUO(SDValue Op,SelectionDAG & DAG) const4966 ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
4967 // Let legalize expand this if it isn't a legal type yet.
4968 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
4969 return SDValue();
4970
4971 SDValue Value, OverflowCmp;
4972 SDValue ARMcc;
4973 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
4974 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4975 SDLoc dl(Op);
4976 // We use 0 and 1 as false and true values.
4977 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4978 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4979 EVT VT = Op.getValueType();
4980
4981 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
4982 ARMcc, CCR, OverflowCmp);
4983
4984 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
4985 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
4986 }
4987
ConvertBooleanCarryToCarryFlag(SDValue BoolCarry,SelectionDAG & DAG)4988 static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry,
4989 SelectionDAG &DAG) {
4990 SDLoc DL(BoolCarry);
4991 EVT CarryVT = BoolCarry.getValueType();
4992
4993 // This converts the boolean value carry into the carry flag by doing
4994 // ARMISD::SUBC Carry, 1
4995 SDValue Carry = DAG.getNode(ARMISD::SUBC, DL,
4996 DAG.getVTList(CarryVT, MVT::i32),
4997 BoolCarry, DAG.getConstant(1, DL, CarryVT));
4998 return Carry.getValue(1);
4999 }
5000
ConvertCarryFlagToBooleanCarry(SDValue Flags,EVT VT,SelectionDAG & DAG)5001 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
5002 SelectionDAG &DAG) {
5003 SDLoc DL(Flags);
5004
5005 // Now convert the carry flag into a boolean carry. We do this
5006 // using ARMISD:ADDE 0, 0, Carry
5007 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
5008 DAG.getConstant(0, DL, MVT::i32),
5009 DAG.getConstant(0, DL, MVT::i32), Flags);
5010 }
5011
LowerUnsignedALUO(SDValue Op,SelectionDAG & DAG) const5012 SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
5013 SelectionDAG &DAG) const {
5014 // Let legalize expand this if it isn't a legal type yet.
5015 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
5016 return SDValue();
5017
5018 SDValue LHS = Op.getOperand(0);
5019 SDValue RHS = Op.getOperand(1);
5020 SDLoc dl(Op);
5021
5022 EVT VT = Op.getValueType();
5023 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
5024 SDValue Value;
5025 SDValue Overflow;
5026 switch (Op.getOpcode()) {
5027 default:
5028 llvm_unreachable("Unknown overflow instruction!");
5029 case ISD::UADDO:
5030 Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS);
5031 // Convert the carry flag into a boolean value.
5032 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
5033 break;
5034 case ISD::USUBO: {
5035 Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS);
5036 // Convert the carry flag into a boolean value.
5037 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
5038 // ARMISD::SUBC returns 0 when we have to borrow, so make it an overflow
5039 // value. So compute 1 - C.
5040 Overflow = DAG.getNode(ISD::SUB, dl, MVT::i32,
5041 DAG.getConstant(1, dl, MVT::i32), Overflow);
5042 break;
5043 }
5044 }
5045
5046 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
5047 }
5048
LowerADDSUBSAT(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)5049 static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG,
5050 const ARMSubtarget *Subtarget) {
5051 EVT VT = Op.getValueType();
5052 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
5053 return SDValue();
5054 if (!VT.isSimple())
5055 return SDValue();
5056
5057 unsigned NewOpcode;
5058 switch (VT.getSimpleVT().SimpleTy) {
5059 default:
5060 return SDValue();
5061 case MVT::i8:
5062 switch (Op->getOpcode()) {
5063 case ISD::UADDSAT:
5064 NewOpcode = ARMISD::UQADD8b;
5065 break;
5066 case ISD::SADDSAT:
5067 NewOpcode = ARMISD::QADD8b;
5068 break;
5069 case ISD::USUBSAT:
5070 NewOpcode = ARMISD::UQSUB8b;
5071 break;
5072 case ISD::SSUBSAT:
5073 NewOpcode = ARMISD::QSUB8b;
5074 break;
5075 }
5076 break;
5077 case MVT::i16:
5078 switch (Op->getOpcode()) {
5079 case ISD::UADDSAT:
5080 NewOpcode = ARMISD::UQADD16b;
5081 break;
5082 case ISD::SADDSAT:
5083 NewOpcode = ARMISD::QADD16b;
5084 break;
5085 case ISD::USUBSAT:
5086 NewOpcode = ARMISD::UQSUB16b;
5087 break;
5088 case ISD::SSUBSAT:
5089 NewOpcode = ARMISD::QSUB16b;
5090 break;
5091 }
5092 break;
5093 }
5094
5095 SDLoc dl(Op);
5096 SDValue Add =
5097 DAG.getNode(NewOpcode, dl, MVT::i32,
5098 DAG.getSExtOrTrunc(Op->getOperand(0), dl, MVT::i32),
5099 DAG.getSExtOrTrunc(Op->getOperand(1), dl, MVT::i32));
5100 return DAG.getNode(ISD::TRUNCATE, dl, VT, Add);
5101 }
5102
LowerSELECT(SDValue Op,SelectionDAG & DAG) const5103 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
5104 SDValue Cond = Op.getOperand(0);
5105 SDValue SelectTrue = Op.getOperand(1);
5106 SDValue SelectFalse = Op.getOperand(2);
5107 SDLoc dl(Op);
5108 unsigned Opc = Cond.getOpcode();
5109
5110 if (Cond.getResNo() == 1 &&
5111 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5112 Opc == ISD::USUBO)) {
5113 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
5114 return SDValue();
5115
5116 SDValue Value, OverflowCmp;
5117 SDValue ARMcc;
5118 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
5119 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5120 EVT VT = Op.getValueType();
5121
5122 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5123 OverflowCmp, DAG);
5124 }
5125
5126 // Convert:
5127 //
5128 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
5129 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
5130 //
5131 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
5132 const ConstantSDNode *CMOVTrue =
5133 dyn_cast<ConstantSDNode>(Cond.getOperand(0));
5134 const ConstantSDNode *CMOVFalse =
5135 dyn_cast<ConstantSDNode>(Cond.getOperand(1));
5136
5137 if (CMOVTrue && CMOVFalse) {
5138 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
5139 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
5140
5141 SDValue True;
5142 SDValue False;
5143 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5144 True = SelectTrue;
5145 False = SelectFalse;
5146 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5147 True = SelectFalse;
5148 False = SelectTrue;
5149 }
5150
5151 if (True.getNode() && False.getNode()) {
5152 EVT VT = Op.getValueType();
5153 SDValue ARMcc = Cond.getOperand(2);
5154 SDValue CCR = Cond.getOperand(3);
5155 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG);
5156 assert(True.getValueType() == VT);
5157 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5158 }
5159 }
5160 }
5161
5162 // ARM's BooleanContents value is UndefinedBooleanContent. Mask out the
5163 // undefined bits before doing a full-word comparison with zero.
5164 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond,
5165 DAG.getConstant(1, dl, Cond.getValueType()));
5166
5167 return DAG.getSelectCC(dl, Cond,
5168 DAG.getConstant(0, dl, Cond.getValueType()),
5169 SelectTrue, SelectFalse, ISD::SETNE);
5170 }
5171
checkVSELConstraints(ISD::CondCode CC,ARMCC::CondCodes & CondCode,bool & swpCmpOps,bool & swpVselOps)5172 static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
5173 bool &swpCmpOps, bool &swpVselOps) {
5174 // Start by selecting the GE condition code for opcodes that return true for
5175 // 'equality'
5176 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
5177 CC == ISD::SETULE || CC == ISD::SETGE || CC == ISD::SETLE)
5178 CondCode = ARMCC::GE;
5179
5180 // and GT for opcodes that return false for 'equality'.
5181 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
5182 CC == ISD::SETULT || CC == ISD::SETGT || CC == ISD::SETLT)
5183 CondCode = ARMCC::GT;
5184
5185 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
5186 // to swap the compare operands.
5187 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
5188 CC == ISD::SETULT || CC == ISD::SETLE || CC == ISD::SETLT)
5189 swpCmpOps = true;
5190
5191 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
5192 // If we have an unordered opcode, we need to swap the operands to the VSEL
5193 // instruction (effectively negating the condition).
5194 //
5195 // This also has the effect of swapping which one of 'less' or 'greater'
5196 // returns true, so we also swap the compare operands. It also switches
5197 // whether we return true for 'equality', so we compensate by picking the
5198 // opposite condition code to our original choice.
5199 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
5200 CC == ISD::SETUGT) {
5201 swpCmpOps = !swpCmpOps;
5202 swpVselOps = !swpVselOps;
5203 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
5204 }
5205
5206 // 'ordered' is 'anything but unordered', so use the VS condition code and
5207 // swap the VSEL operands.
5208 if (CC == ISD::SETO) {
5209 CondCode = ARMCC::VS;
5210 swpVselOps = true;
5211 }
5212
5213 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
5214 // code and swap the VSEL operands. Also do this if we don't care about the
5215 // unordered case.
5216 if (CC == ISD::SETUNE || CC == ISD::SETNE) {
5217 CondCode = ARMCC::EQ;
5218 swpVselOps = true;
5219 }
5220 }
5221
getCMOV(const SDLoc & dl,EVT VT,SDValue FalseVal,SDValue TrueVal,SDValue ARMcc,SDValue CCR,SDValue Cmp,SelectionDAG & DAG) const5222 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
5223 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
5224 SDValue Cmp, SelectionDAG &DAG) const {
5225 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5226 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
5227 DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
5228 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl,
5229 DAG.getVTList(MVT::i32, MVT::i32), TrueVal);
5230
5231 SDValue TrueLow = TrueVal.getValue(0);
5232 SDValue TrueHigh = TrueVal.getValue(1);
5233 SDValue FalseLow = FalseVal.getValue(0);
5234 SDValue FalseHigh = FalseVal.getValue(1);
5235
5236 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
5237 ARMcc, CCR, Cmp);
5238 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
5239 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5240
5241 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
5242 } else {
5243 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR,
5244 Cmp);
5245 }
5246 }
5247
isGTorGE(ISD::CondCode CC)5248 static bool isGTorGE(ISD::CondCode CC) {
5249 return CC == ISD::SETGT || CC == ISD::SETGE;
5250 }
5251
isLTorLE(ISD::CondCode CC)5252 static bool isLTorLE(ISD::CondCode CC) {
5253 return CC == ISD::SETLT || CC == ISD::SETLE;
5254 }
5255
5256 // See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
5257 // All of these conditions (and their <= and >= counterparts) will do:
5258 // x < k ? k : x
5259 // x > k ? x : k
5260 // k < x ? x : k
5261 // k > x ? k : x
isLowerSaturate(const SDValue LHS,const SDValue RHS,const SDValue TrueVal,const SDValue FalseVal,const ISD::CondCode CC,const SDValue K)5262 static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
5263 const SDValue TrueVal, const SDValue FalseVal,
5264 const ISD::CondCode CC, const SDValue K) {
5265 return (isGTorGE(CC) &&
5266 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
5267 (isLTorLE(CC) &&
5268 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
5269 }
5270
5271 // Check if two chained conditionals could be converted into SSAT or USAT.
5272 //
5273 // SSAT can replace a set of two conditional selectors that bound a number to an
5274 // interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
5275 //
5276 // x < -k ? -k : (x > k ? k : x)
5277 // x < -k ? -k : (x < k ? x : k)
5278 // x > -k ? (x > k ? k : x) : -k
5279 // x < k ? (x < -k ? -k : x) : k
5280 // etc.
5281 //
5282 // LLVM canonicalizes these to either a min(max()) or a max(min())
5283 // pattern. This function tries to match one of these and will return a SSAT
5284 // node if successful.
5285 //
5286 // USAT works similarily to SSAT but bounds on the interval [0, k] where k + 1
5287 // is a power of 2.
LowerSaturatingConditional(SDValue Op,SelectionDAG & DAG)5288 static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
5289 EVT VT = Op.getValueType();
5290 SDValue V1 = Op.getOperand(0);
5291 SDValue K1 = Op.getOperand(1);
5292 SDValue TrueVal1 = Op.getOperand(2);
5293 SDValue FalseVal1 = Op.getOperand(3);
5294 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5295
5296 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5297 if (Op2.getOpcode() != ISD::SELECT_CC)
5298 return SDValue();
5299
5300 SDValue V2 = Op2.getOperand(0);
5301 SDValue K2 = Op2.getOperand(1);
5302 SDValue TrueVal2 = Op2.getOperand(2);
5303 SDValue FalseVal2 = Op2.getOperand(3);
5304 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get();
5305
5306 SDValue V1Tmp = V1;
5307 SDValue V2Tmp = V2;
5308
5309 // Check that the registers and the constants match a max(min()) or min(max())
5310 // pattern
5311 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5312 K2 != FalseVal2 ||
5313 !((isGTorGE(CC1) && isLTorLE(CC2)) || (isLTorLE(CC1) && isGTorGE(CC2))))
5314 return SDValue();
5315
5316 // Check that the constant in the lower-bound check is
5317 // the opposite of the constant in the upper-bound check
5318 // in 1's complement.
5319 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5320 return SDValue();
5321
5322 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5323 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5324 int64_t PosVal = std::max(Val1, Val2);
5325 int64_t NegVal = std::min(Val1, Val2);
5326
5327 if (!((Val1 > Val2 && isLTorLE(CC1)) || (Val1 < Val2 && isLTorLE(CC2))) ||
5328 !isPowerOf2_64(PosVal + 1))
5329 return SDValue();
5330
5331 // Handle the difference between USAT (unsigned) and SSAT (signed)
5332 // saturation
5333 // At this point, PosVal is guaranteed to be positive
5334 uint64_t K = PosVal;
5335 SDLoc dl(Op);
5336 if (Val1 == ~Val2)
5337 return DAG.getNode(ARMISD::SSAT, dl, VT, V2Tmp,
5338 DAG.getConstant(countTrailingOnes(K), dl, VT));
5339 if (NegVal == 0)
5340 return DAG.getNode(ARMISD::USAT, dl, VT, V2Tmp,
5341 DAG.getConstant(countTrailingOnes(K), dl, VT));
5342
5343 return SDValue();
5344 }
5345
5346 // Check if a condition of the type x < k ? k : x can be converted into a
5347 // bit operation instead of conditional moves.
5348 // Currently this is allowed given:
5349 // - The conditions and values match up
5350 // - k is 0 or -1 (all ones)
5351 // This function will not check the last condition, thats up to the caller
5352 // It returns true if the transformation can be made, and in such case
5353 // returns x in V, and k in SatK.
isLowerSaturatingConditional(const SDValue & Op,SDValue & V,SDValue & SatK)5354 static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
5355 SDValue &SatK)
5356 {
5357 SDValue LHS = Op.getOperand(0);
5358 SDValue RHS = Op.getOperand(1);
5359 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5360 SDValue TrueVal = Op.getOperand(2);
5361 SDValue FalseVal = Op.getOperand(3);
5362
5363 SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS : isa<ConstantSDNode>(RHS)
5364 ? &RHS
5365 : nullptr;
5366
5367 // No constant operation in comparison, early out
5368 if (!K)
5369 return false;
5370
5371 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5372 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5373 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
5374
5375 // If the constant on left and right side, or variable on left and right,
5376 // does not match, early out
5377 if (*K != KTmp || V != VTmp)
5378 return false;
5379
5380 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, *K)) {
5381 SatK = *K;
5382 return true;
5383 }
5384
5385 return false;
5386 }
5387
isUnsupportedFloatingType(EVT VT) const5388 bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const {
5389 if (VT == MVT::f32)
5390 return !Subtarget->hasVFP2Base();
5391 if (VT == MVT::f64)
5392 return !Subtarget->hasFP64();
5393 if (VT == MVT::f16)
5394 return !Subtarget->hasFullFP16();
5395 return false;
5396 }
5397
LowerSELECT_CC(SDValue Op,SelectionDAG & DAG) const5398 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5399 EVT VT = Op.getValueType();
5400 SDLoc dl(Op);
5401
5402 // Try to convert two saturating conditional selects into a single SSAT
5403 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
5404 if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
5405 return SatValue;
5406
5407 // Try to convert expressions of the form x < k ? k : x (and similar forms)
5408 // into more efficient bit operations, which is possible when k is 0 or -1
5409 // On ARM and Thumb-2 which have flexible operand 2 this will result in
5410 // single instructions. On Thumb the shift and the bit operation will be two
5411 // instructions.
5412 // Only allow this transformation on full-width (32-bit) operations
5413 SDValue LowerSatConstant;
5414 SDValue SatValue;
5415 if (VT == MVT::i32 &&
5416 isLowerSaturatingConditional(Op, SatValue, LowerSatConstant)) {
5417 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue,
5418 DAG.getConstant(31, dl, VT));
5419 if (isNullConstant(LowerSatConstant)) {
5420 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV,
5421 DAG.getAllOnesConstant(dl, VT));
5422 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV);
5423 } else if (isAllOnesConstant(LowerSatConstant))
5424 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV);
5425 }
5426
5427 SDValue LHS = Op.getOperand(0);
5428 SDValue RHS = Op.getOperand(1);
5429 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5430 SDValue TrueVal = Op.getOperand(2);
5431 SDValue FalseVal = Op.getOperand(3);
5432 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
5433 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
5434
5435 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5436 LHS.getValueType() == MVT::i32 && RHS.getValueType() == MVT::i32) {
5437 unsigned TVal = CTVal->getZExtValue();
5438 unsigned FVal = CFVal->getZExtValue();
5439 unsigned Opcode = 0;
5440
5441 if (TVal == ~FVal) {
5442 Opcode = ARMISD::CSINV;
5443 } else if (TVal == ~FVal + 1) {
5444 Opcode = ARMISD::CSNEG;
5445 } else if (TVal + 1 == FVal) {
5446 Opcode = ARMISD::CSINC;
5447 } else if (TVal == FVal + 1) {
5448 Opcode = ARMISD::CSINC;
5449 std::swap(TrueVal, FalseVal);
5450 std::swap(TVal, FVal);
5451 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5452 }
5453
5454 if (Opcode) {
5455 // If one of the constants is cheaper than another, materialise the
5456 // cheaper one and let the csel generate the other.
5457 if (Opcode != ARMISD::CSINC &&
5458 HasLowerConstantMaterializationCost(FVal, TVal, Subtarget)) {
5459 std::swap(TrueVal, FalseVal);
5460 std::swap(TVal, FVal);
5461 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5462 }
5463
5464 // Attempt to use ZR checking TVal is 0, possibly inverting the condition
5465 // to get there. CSINC not is invertable like the other two (~(~a) == a,
5466 // -(-a) == a, but (a+1)+1 != a).
5467 if (FVal == 0 && Opcode != ARMISD::CSINC) {
5468 std::swap(TrueVal, FalseVal);
5469 std::swap(TVal, FVal);
5470 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5471 }
5472
5473 // Drops F's value because we can get it by inverting/negating TVal.
5474 FalseVal = TrueVal;
5475
5476 SDValue ARMcc;
5477 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5478 EVT VT = TrueVal.getValueType();
5479 return DAG.getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5480 }
5481 }
5482
5483 if (isUnsupportedFloatingType(LHS.getValueType())) {
5484 DAG.getTargetLoweringInfo().softenSetCCOperands(
5485 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5486
5487 // If softenSetCCOperands only returned one value, we should compare it to
5488 // zero.
5489 if (!RHS.getNode()) {
5490 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5491 CC = ISD::SETNE;
5492 }
5493 }
5494
5495 if (LHS.getValueType() == MVT::i32) {
5496 // Try to generate VSEL on ARMv8.
5497 // The VSEL instruction can't use all the usual ARM condition
5498 // codes: it only has two bits to select the condition code, so it's
5499 // constrained to use only GE, GT, VS and EQ.
5500 //
5501 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
5502 // swap the operands of the previous compare instruction (effectively
5503 // inverting the compare condition, swapping 'less' and 'greater') and
5504 // sometimes need to swap the operands to the VSEL (which inverts the
5505 // condition in the sense of firing whenever the previous condition didn't)
5506 if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 ||
5507 TrueVal.getValueType() == MVT::f32 ||
5508 TrueVal.getValueType() == MVT::f64)) {
5509 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5510 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
5511 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
5512 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5513 std::swap(TrueVal, FalseVal);
5514 }
5515 }
5516
5517 SDValue ARMcc;
5518 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5519 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5520 // Choose GE over PL, which vsel does now support
5521 if (cast<ConstantSDNode>(ARMcc)->getZExtValue() == ARMCC::PL)
5522 ARMcc = DAG.getConstant(ARMCC::GE, dl, MVT::i32);
5523 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5524 }
5525
5526 ARMCC::CondCodes CondCode, CondCode2;
5527 FPCCToARMCC(CC, CondCode, CondCode2);
5528
5529 // Normalize the fp compare. If RHS is zero we prefer to keep it there so we
5530 // match CMPFPw0 instead of CMPFP, though we don't do this for f16 because we
5531 // must use VSEL (limited condition codes), due to not having conditional f16
5532 // moves.
5533 if (Subtarget->hasFPARMv8Base() &&
5534 !(isFloatingPointZero(RHS) && TrueVal.getValueType() != MVT::f16) &&
5535 (TrueVal.getValueType() == MVT::f16 ||
5536 TrueVal.getValueType() == MVT::f32 ||
5537 TrueVal.getValueType() == MVT::f64)) {
5538 bool swpCmpOps = false;
5539 bool swpVselOps = false;
5540 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
5541
5542 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
5543 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
5544 if (swpCmpOps)
5545 std::swap(LHS, RHS);
5546 if (swpVselOps)
5547 std::swap(TrueVal, FalseVal);
5548 }
5549 }
5550
5551 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5552 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5553 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5554 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5555 if (CondCode2 != ARMCC::AL) {
5556 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
5557 // FIXME: Needs another CMP because flag can have but one use.
5558 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5559 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5560 }
5561 return Result;
5562 }
5563
5564 /// canChangeToInt - Given the fp compare operand, return true if it is suitable
5565 /// to morph to an integer compare sequence.
canChangeToInt(SDValue Op,bool & SeenZero,const ARMSubtarget * Subtarget)5566 static bool canChangeToInt(SDValue Op, bool &SeenZero,
5567 const ARMSubtarget *Subtarget) {
5568 SDNode *N = Op.getNode();
5569 if (!N->hasOneUse())
5570 // Otherwise it requires moving the value from fp to integer registers.
5571 return false;
5572 if (!N->getNumValues())
5573 return false;
5574 EVT VT = Op.getValueType();
5575 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5576 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
5577 // vmrs are very slow, e.g. cortex-a8.
5578 return false;
5579
5580 if (isFloatingPointZero(Op)) {
5581 SeenZero = true;
5582 return true;
5583 }
5584 return ISD::isNormalLoad(N);
5585 }
5586
bitcastf32Toi32(SDValue Op,SelectionDAG & DAG)5587 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5588 if (isFloatingPointZero(Op))
5589 return DAG.getConstant(0, SDLoc(Op), MVT::i32);
5590
5591 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op))
5592 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(),
5593 Ld->getPointerInfo(), Ld->getAlign(),
5594 Ld->getMemOperand()->getFlags());
5595
5596 llvm_unreachable("Unknown VFP cmp argument!");
5597 }
5598
expandf64Toi32(SDValue Op,SelectionDAG & DAG,SDValue & RetVal1,SDValue & RetVal2)5599 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5600 SDValue &RetVal1, SDValue &RetVal2) {
5601 SDLoc dl(Op);
5602
5603 if (isFloatingPointZero(Op)) {
5604 RetVal1 = DAG.getConstant(0, dl, MVT::i32);
5605 RetVal2 = DAG.getConstant(0, dl, MVT::i32);
5606 return;
5607 }
5608
5609 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) {
5610 SDValue Ptr = Ld->getBasePtr();
5611 RetVal1 =
5612 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
5613 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5614
5615 EVT PtrType = Ptr.getValueType();
5616 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
5617 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
5618 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5619 Ld->getPointerInfo().getWithOffset(4),
5620 commonAlignment(Ld->getAlign(), 4),
5621 Ld->getMemOperand()->getFlags());
5622 return;
5623 }
5624
5625 llvm_unreachable("Unknown VFP cmp argument!");
5626 }
5627
5628 /// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some
5629 /// f32 and even f64 comparisons to integer ones.
5630 SDValue
OptimizeVFPBrcond(SDValue Op,SelectionDAG & DAG) const5631 ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5632 SDValue Chain = Op.getOperand(0);
5633 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
5634 SDValue LHS = Op.getOperand(2);
5635 SDValue RHS = Op.getOperand(3);
5636 SDValue Dest = Op.getOperand(4);
5637 SDLoc dl(Op);
5638
5639 bool LHSSeenZero = false;
5640 bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget);
5641 bool RHSSeenZero = false;
5642 bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget);
5643 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5644 // If unsafe fp math optimization is enabled and there are no other uses of
5645 // the CMP operands, and the condition code is EQ or NE, we can optimize it
5646 // to an integer comparison.
5647 if (CC == ISD::SETOEQ)
5648 CC = ISD::SETEQ;
5649 else if (CC == ISD::SETUNE)
5650 CC = ISD::SETNE;
5651
5652 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5653 SDValue ARMcc;
5654 if (LHS.getValueType() == MVT::f32) {
5655 LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5656 bitcastf32Toi32(LHS, DAG), Mask);
5657 RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5658 bitcastf32Toi32(RHS, DAG), Mask);
5659 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5660 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5661 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
5662 Chain, Dest, ARMcc, CCR, Cmp);
5663 }
5664
5665 SDValue LHS1, LHS2;
5666 SDValue RHS1, RHS2;
5667 expandf64Toi32(LHS, DAG, LHS1, LHS2);
5668 expandf64Toi32(RHS, DAG, RHS1, RHS2);
5669 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
5670 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
5671 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5672 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5673 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
5674 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5675 return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops);
5676 }
5677
5678 return SDValue();
5679 }
5680
LowerBRCOND(SDValue Op,SelectionDAG & DAG) const5681 SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5682 SDValue Chain = Op.getOperand(0);
5683 SDValue Cond = Op.getOperand(1);
5684 SDValue Dest = Op.getOperand(2);
5685 SDLoc dl(Op);
5686
5687 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5688 // instruction.
5689 unsigned Opc = Cond.getOpcode();
5690 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5691 !Subtarget->isThumb1Only();
5692 if (Cond.getResNo() == 1 &&
5693 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5694 Opc == ISD::USUBO || OptimizeMul)) {
5695 // Only lower legal XALUO ops.
5696 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
5697 return SDValue();
5698
5699 // The actual operation with overflow check.
5700 SDValue Value, OverflowCmp;
5701 SDValue ARMcc;
5702 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
5703
5704 // Reverse the condition code.
5705 ARMCC::CondCodes CondCode =
5706 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue();
5707 CondCode = ARMCC::getOppositeCondition(CondCode);
5708 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
5709 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5710
5711 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR,
5712 OverflowCmp);
5713 }
5714
5715 return SDValue();
5716 }
5717
LowerBR_CC(SDValue Op,SelectionDAG & DAG) const5718 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5719 SDValue Chain = Op.getOperand(0);
5720 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
5721 SDValue LHS = Op.getOperand(2);
5722 SDValue RHS = Op.getOperand(3);
5723 SDValue Dest = Op.getOperand(4);
5724 SDLoc dl(Op);
5725
5726 if (isUnsupportedFloatingType(LHS.getValueType())) {
5727 DAG.getTargetLoweringInfo().softenSetCCOperands(
5728 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5729
5730 // If softenSetCCOperands only returned one value, we should compare it to
5731 // zero.
5732 if (!RHS.getNode()) {
5733 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5734 CC = ISD::SETNE;
5735 }
5736 }
5737
5738 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5739 // instruction.
5740 unsigned Opc = LHS.getOpcode();
5741 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5742 !Subtarget->isThumb1Only();
5743 if (LHS.getResNo() == 1 && (isOneConstant(RHS) || isNullConstant(RHS)) &&
5744 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5745 Opc == ISD::USUBO || OptimizeMul) &&
5746 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
5747 // Only lower legal XALUO ops.
5748 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
5749 return SDValue();
5750
5751 // The actual operation with overflow check.
5752 SDValue Value, OverflowCmp;
5753 SDValue ARMcc;
5754 std::tie(Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc);
5755
5756 if ((CC == ISD::SETNE) != isOneConstant(RHS)) {
5757 // Reverse the condition code.
5758 ARMCC::CondCodes CondCode =
5759 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue();
5760 CondCode = ARMCC::getOppositeCondition(CondCode);
5761 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
5762 }
5763 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5764
5765 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR,
5766 OverflowCmp);
5767 }
5768
5769 if (LHS.getValueType() == MVT::i32) {
5770 SDValue ARMcc;
5771 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5772 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5773 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
5774 Chain, Dest, ARMcc, CCR, Cmp);
5775 }
5776
5777 if (getTargetMachine().Options.UnsafeFPMath &&
5778 (CC == ISD::SETEQ || CC == ISD::SETOEQ ||
5779 CC == ISD::SETNE || CC == ISD::SETUNE)) {
5780 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5781 return Result;
5782 }
5783
5784 ARMCC::CondCodes CondCode, CondCode2;
5785 FPCCToARMCC(CC, CondCode, CondCode2);
5786
5787 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5788 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5789 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5790 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
5791 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
5792 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
5793 if (CondCode2 != ARMCC::AL) {
5794 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
5795 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
5796 Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
5797 }
5798 return Res;
5799 }
5800
LowerBR_JT(SDValue Op,SelectionDAG & DAG) const5801 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5802 SDValue Chain = Op.getOperand(0);
5803 SDValue Table = Op.getOperand(1);
5804 SDValue Index = Op.getOperand(2);
5805 SDLoc dl(Op);
5806
5807 EVT PTy = getPointerTy(DAG.getDataLayout());
5808 JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
5809 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
5810 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
5811 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
5812 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index);
5813 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5814 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table
5815 // which does another jump to the destination. This also makes it easier
5816 // to translate it to TBB / TBH later (Thumb2 only).
5817 // FIXME: This might not work if the function is extremely large.
5818 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
5819 Addr, Op.getOperand(2), JTI);
5820 }
5821 if (isPositionIndependent() || Subtarget->isROPI()) {
5822 Addr =
5823 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
5824 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
5825 Chain = Addr.getValue(1);
5826 Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr);
5827 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5828 } else {
5829 Addr =
5830 DAG.getLoad(PTy, dl, Chain, Addr,
5831 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
5832 Chain = Addr.getValue(1);
5833 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5834 }
5835 }
5836
LowerVectorFP_TO_INT(SDValue Op,SelectionDAG & DAG)5837 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5838 EVT VT = Op.getValueType();
5839 SDLoc dl(Op);
5840
5841 if (Op.getValueType().getVectorElementType() == MVT::i32) {
5842 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5843 return Op;
5844 return DAG.UnrollVectorOp(Op.getNode());
5845 }
5846
5847 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5848
5849 EVT NewTy;
5850 const EVT OpTy = Op.getOperand(0).getValueType();
5851 if (OpTy == MVT::v4f32)
5852 NewTy = MVT::v4i32;
5853 else if (OpTy == MVT::v4f16 && HasFullFP16)
5854 NewTy = MVT::v4i16;
5855 else if (OpTy == MVT::v8f16 && HasFullFP16)
5856 NewTy = MVT::v8i16;
5857 else
5858 llvm_unreachable("Invalid type for custom lowering!");
5859
5860 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5861 return DAG.UnrollVectorOp(Op.getNode());
5862
5863 Op = DAG.getNode(Op.getOpcode(), dl, NewTy, Op.getOperand(0));
5864 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
5865 }
5866
LowerFP_TO_INT(SDValue Op,SelectionDAG & DAG) const5867 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5868 EVT VT = Op.getValueType();
5869 if (VT.isVector())
5870 return LowerVectorFP_TO_INT(Op, DAG);
5871
5872 bool IsStrict = Op->isStrictFPOpcode();
5873 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
5874
5875 if (isUnsupportedFloatingType(SrcVal.getValueType())) {
5876 RTLIB::Libcall LC;
5877 if (Op.getOpcode() == ISD::FP_TO_SINT ||
5878 Op.getOpcode() == ISD::STRICT_FP_TO_SINT)
5879 LC = RTLIB::getFPTOSINT(SrcVal.getValueType(),
5880 Op.getValueType());
5881 else
5882 LC = RTLIB::getFPTOUINT(SrcVal.getValueType(),
5883 Op.getValueType());
5884 SDLoc Loc(Op);
5885 MakeLibCallOptions CallOptions;
5886 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
5887 SDValue Result;
5888 std::tie(Result, Chain) = makeLibCall(DAG, LC, Op.getValueType(), SrcVal,
5889 CallOptions, Loc, Chain);
5890 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
5891 }
5892
5893 // FIXME: Remove this when we have strict fp instruction selection patterns
5894 if (IsStrict) {
5895 SDLoc Loc(Op);
5896 SDValue Result =
5897 DAG.getNode(Op.getOpcode() == ISD::STRICT_FP_TO_SINT ? ISD::FP_TO_SINT
5898 : ISD::FP_TO_UINT,
5899 Loc, Op.getValueType(), SrcVal);
5900 return DAG.getMergeValues({Result, Op.getOperand(0)}, Loc);
5901 }
5902
5903 return Op;
5904 }
5905
LowerFP_TO_INT_SAT(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)5906 static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
5907 const ARMSubtarget *Subtarget) {
5908 EVT VT = Op.getValueType();
5909 EVT ToVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
5910 EVT FromVT = Op.getOperand(0).getValueType();
5911
5912 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5913 return Op;
5914 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5915 Subtarget->hasFP64())
5916 return Op;
5917 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5918 Subtarget->hasFullFP16())
5919 return Op;
5920 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5921 Subtarget->hasMVEFloatOps())
5922 return Op;
5923 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5924 Subtarget->hasMVEFloatOps())
5925 return Op;
5926
5927 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5928 return SDValue();
5929
5930 SDLoc DL(Op);
5931 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
5932 unsigned BW = ToVT.getScalarSizeInBits() - IsSigned;
5933 SDValue CVT = DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
5934 DAG.getValueType(VT.getScalarType()));
5935 SDValue Max = DAG.getNode(IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, CVT,
5936 DAG.getConstant((1 << BW) - 1, DL, VT));
5937 if (IsSigned)
5938 Max = DAG.getNode(ISD::SMAX, DL, VT, Max,
5939 DAG.getConstant(-(1 << BW), DL, VT));
5940 return Max;
5941 }
5942
LowerVectorINT_TO_FP(SDValue Op,SelectionDAG & DAG)5943 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5944 EVT VT = Op.getValueType();
5945 SDLoc dl(Op);
5946
5947 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5948 if (VT.getVectorElementType() == MVT::f32)
5949 return Op;
5950 return DAG.UnrollVectorOp(Op.getNode());
5951 }
5952
5953 assert((Op.getOperand(0).getValueType() == MVT::v4i16 ||
5954 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5955 "Invalid type for custom lowering!");
5956
5957 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5958
5959 EVT DestVecType;
5960 if (VT == MVT::v4f32)
5961 DestVecType = MVT::v4i32;
5962 else if (VT == MVT::v4f16 && HasFullFP16)
5963 DestVecType = MVT::v4i16;
5964 else if (VT == MVT::v8f16 && HasFullFP16)
5965 DestVecType = MVT::v8i16;
5966 else
5967 return DAG.UnrollVectorOp(Op.getNode());
5968
5969 unsigned CastOpc;
5970 unsigned Opc;
5971 switch (Op.getOpcode()) {
5972 default: llvm_unreachable("Invalid opcode!");
5973 case ISD::SINT_TO_FP:
5974 CastOpc = ISD::SIGN_EXTEND;
5975 Opc = ISD::SINT_TO_FP;
5976 break;
5977 case ISD::UINT_TO_FP:
5978 CastOpc = ISD::ZERO_EXTEND;
5979 Opc = ISD::UINT_TO_FP;
5980 break;
5981 }
5982
5983 Op = DAG.getNode(CastOpc, dl, DestVecType, Op.getOperand(0));
5984 return DAG.getNode(Opc, dl, VT, Op);
5985 }
5986
LowerINT_TO_FP(SDValue Op,SelectionDAG & DAG) const5987 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5988 EVT VT = Op.getValueType();
5989 if (VT.isVector())
5990 return LowerVectorINT_TO_FP(Op, DAG);
5991 if (isUnsupportedFloatingType(VT)) {
5992 RTLIB::Libcall LC;
5993 if (Op.getOpcode() == ISD::SINT_TO_FP)
5994 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(),
5995 Op.getValueType());
5996 else
5997 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(),
5998 Op.getValueType());
5999 MakeLibCallOptions CallOptions;
6000 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0),
6001 CallOptions, SDLoc(Op)).first;
6002 }
6003
6004 return Op;
6005 }
6006
LowerFCOPYSIGN(SDValue Op,SelectionDAG & DAG) const6007 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
6008 // Implement fcopysign with a fabs and a conditional fneg.
6009 SDValue Tmp0 = Op.getOperand(0);
6010 SDValue Tmp1 = Op.getOperand(1);
6011 SDLoc dl(Op);
6012 EVT VT = Op.getValueType();
6013 EVT SrcVT = Tmp1.getValueType();
6014 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
6015 Tmp0.getOpcode() == ARMISD::VMOVDRR;
6016 bool UseNEON = !InGPR && Subtarget->hasNEON();
6017
6018 if (UseNEON) {
6019 // Use VBSL to copy the sign bit.
6020 unsigned EncodedVal = ARM_AM::createVMOVModImm(0x6, 0x80);
6021 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
6022 DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
6023 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6024 if (VT == MVT::f64)
6025 Mask = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
6026 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
6027 DAG.getConstant(32, dl, MVT::i32));
6028 else /*if (VT == MVT::f32)*/
6029 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
6030 if (SrcVT == MVT::f32) {
6031 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1);
6032 if (VT == MVT::f64)
6033 Tmp1 = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
6034 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
6035 DAG.getConstant(32, dl, MVT::i32));
6036 } else if (VT == MVT::f32)
6037 Tmp1 = DAG.getNode(ARMISD::VSHRuIMM, dl, MVT::v1i64,
6038 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
6039 DAG.getConstant(32, dl, MVT::i32));
6040 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
6041 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
6042
6043 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff),
6044 dl, MVT::i32);
6045 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
6046 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
6047 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
6048
6049 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
6050 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask),
6051 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot));
6052 if (VT == MVT::f32) {
6053 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
6054 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
6055 DAG.getConstant(0, dl, MVT::i32));
6056 } else {
6057 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
6058 }
6059
6060 return Res;
6061 }
6062
6063 // Bitcast operand 1 to i32.
6064 if (SrcVT == MVT::f64)
6065 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
6066 Tmp1).getValue(1);
6067 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
6068
6069 // Or in the signbit with integer operations.
6070 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
6071 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
6072 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
6073 if (VT == MVT::f32) {
6074 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
6075 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
6076 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
6077 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1));
6078 }
6079
6080 // f64: Or the high part with signbit and then combine two parts.
6081 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
6082 Tmp0);
6083 SDValue Lo = Tmp0.getValue(0);
6084 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
6085 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1);
6086 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
6087 }
6088
LowerRETURNADDR(SDValue Op,SelectionDAG & DAG) const6089 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
6090 MachineFunction &MF = DAG.getMachineFunction();
6091 MachineFrameInfo &MFI = MF.getFrameInfo();
6092 MFI.setReturnAddressIsTaken(true);
6093
6094 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
6095 return SDValue();
6096
6097 EVT VT = Op.getValueType();
6098 SDLoc dl(Op);
6099 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
6100 if (Depth) {
6101 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
6102 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
6103 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
6104 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
6105 MachinePointerInfo());
6106 }
6107
6108 // Return LR, which contains the return address. Mark it an implicit live-in.
6109 Register Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
6110 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
6111 }
6112
LowerFRAMEADDR(SDValue Op,SelectionDAG & DAG) const6113 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
6114 const ARMBaseRegisterInfo &ARI =
6115 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
6116 MachineFunction &MF = DAG.getMachineFunction();
6117 MachineFrameInfo &MFI = MF.getFrameInfo();
6118 MFI.setFrameAddressIsTaken(true);
6119
6120 EVT VT = Op.getValueType();
6121 SDLoc dl(Op); // FIXME probably not meaningful
6122 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
6123 Register FrameReg = ARI.getFrameRegister(MF);
6124 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
6125 while (Depth--)
6126 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
6127 MachinePointerInfo());
6128 return FrameAddr;
6129 }
6130
6131 // FIXME? Maybe this could be a TableGen attribute on some registers and
6132 // this table could be generated automatically from RegInfo.
getRegisterByName(const char * RegName,LLT VT,const MachineFunction & MF) const6133 Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT,
6134 const MachineFunction &MF) const {
6135 Register Reg = StringSwitch<unsigned>(RegName)
6136 .Case("sp", ARM::SP)
6137 .Default(0);
6138 if (Reg)
6139 return Reg;
6140 report_fatal_error(Twine("Invalid register name \""
6141 + StringRef(RegName) + "\"."));
6142 }
6143
6144 // Result is 64 bit value so split into two 32 bit values and return as a
6145 // pair of values.
ExpandREAD_REGISTER(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG)6146 static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
6147 SelectionDAG &DAG) {
6148 SDLoc DL(N);
6149
6150 // This function is only supposed to be called for i64 type destination.
6151 assert(N->getValueType(0) == MVT::i64
6152 && "ExpandREAD_REGISTER called for non-i64 type result.");
6153
6154 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
6155 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
6156 N->getOperand(0),
6157 N->getOperand(1));
6158
6159 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
6160 Read.getValue(1)));
6161 Results.push_back(Read.getOperand(0));
6162 }
6163
6164 /// \p BC is a bitcast that is about to be turned into a VMOVDRR.
6165 /// When \p DstVT, the destination type of \p BC, is on the vector
6166 /// register bank and the source of bitcast, \p Op, operates on the same bank,
6167 /// it might be possible to combine them, such that everything stays on the
6168 /// vector register bank.
6169 /// \p return The node that would replace \p BT, if the combine
6170 /// is possible.
CombineVMOVDRRCandidateWithVecOp(const SDNode * BC,SelectionDAG & DAG)6171 static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
6172 SelectionDAG &DAG) {
6173 SDValue Op = BC->getOperand(0);
6174 EVT DstVT = BC->getValueType(0);
6175
6176 // The only vector instruction that can produce a scalar (remember,
6177 // since the bitcast was about to be turned into VMOVDRR, the source
6178 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
6179 // Moreover, we can do this combine only if there is one use.
6180 // Finally, if the destination type is not a vector, there is not
6181 // much point on forcing everything on the vector bank.
6182 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6183 !Op.hasOneUse())
6184 return SDValue();
6185
6186 // If the index is not constant, we will introduce an additional
6187 // multiply that will stick.
6188 // Give up in that case.
6189 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6190 if (!Index)
6191 return SDValue();
6192 unsigned DstNumElt = DstVT.getVectorNumElements();
6193
6194 // Compute the new index.
6195 const APInt &APIntIndex = Index->getAPIntValue();
6196 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
6197 NewIndex *= APIntIndex;
6198 // Check if the new constant index fits into i32.
6199 if (NewIndex.getBitWidth() > 32)
6200 return SDValue();
6201
6202 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
6203 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
6204 SDLoc dl(Op);
6205 SDValue ExtractSrc = Op.getOperand(0);
6206 EVT VecVT = EVT::getVectorVT(
6207 *DAG.getContext(), DstVT.getScalarType(),
6208 ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
6209 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
6210 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast,
6211 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32));
6212 }
6213
6214 /// ExpandBITCAST - If the target supports VFP, this function is called to
6215 /// expand a bit convert where either the source or destination type is i64 to
6216 /// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
6217 /// operand type is illegal (e.g., v2f32 for a target that doesn't support
6218 /// vectors), since the legalizer won't know what to do with that.
ExpandBITCAST(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget) const6219 SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
6220 const ARMSubtarget *Subtarget) const {
6221 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6222 SDLoc dl(N);
6223 SDValue Op = N->getOperand(0);
6224
6225 // This function is only supposed to be called for i16 and i64 types, either
6226 // as the source or destination of the bit convert.
6227 EVT SrcVT = Op.getValueType();
6228 EVT DstVT = N->getValueType(0);
6229
6230 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6231 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6232 return MoveToHPR(SDLoc(N), DAG, MVT::i32, DstVT.getSimpleVT(),
6233 DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op));
6234
6235 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6236 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6237 return DAG.getNode(
6238 ISD::TRUNCATE, SDLoc(N), DstVT,
6239 MoveFromHPR(SDLoc(N), DAG, MVT::i32, SrcVT.getSimpleVT(), Op));
6240
6241 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6242 return SDValue();
6243
6244 // Turn i64->f64 into VMOVDRR.
6245 if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) {
6246 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
6247 // if we can combine the bitcast with its source.
6248 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
6249 return Val;
6250
6251 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
6252 DAG.getConstant(0, dl, MVT::i32));
6253 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
6254 DAG.getConstant(1, dl, MVT::i32));
6255 return DAG.getNode(ISD::BITCAST, dl, DstVT,
6256 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
6257 }
6258
6259 // Turn f64->i64 into VMOVRRD.
6260 if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) {
6261 SDValue Cvt;
6262 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
6263 SrcVT.getVectorNumElements() > 1)
6264 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
6265 DAG.getVTList(MVT::i32, MVT::i32),
6266 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op));
6267 else
6268 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
6269 DAG.getVTList(MVT::i32, MVT::i32), Op);
6270 // Merge the pieces into a single i64 value.
6271 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
6272 }
6273
6274 return SDValue();
6275 }
6276
6277 /// getZeroVector - Returns a vector of specified type with all zero elements.
6278 /// Zero vectors are used to represent vector negation and in those cases
6279 /// will be implemented with the NEON VNEG instruction. However, VNEG does
6280 /// not support i64 elements, so sometimes the zero vectors will need to be
6281 /// explicitly constructed. Regardless, use a canonical VMOV to create the
6282 /// zero vector.
getZeroVector(EVT VT,SelectionDAG & DAG,const SDLoc & dl)6283 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6284 assert(VT.isVector() && "Expected a vector type");
6285 // The canonical modified immediate encoding of a zero vector is....0!
6286 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
6287 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
6288 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
6289 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
6290 }
6291
6292 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two
6293 /// i32 values and take a 2 x i32 value to shift plus a shift amount.
LowerShiftRightParts(SDValue Op,SelectionDAG & DAG) const6294 SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
6295 SelectionDAG &DAG) const {
6296 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6297 EVT VT = Op.getValueType();
6298 unsigned VTBits = VT.getSizeInBits();
6299 SDLoc dl(Op);
6300 SDValue ShOpLo = Op.getOperand(0);
6301 SDValue ShOpHi = Op.getOperand(1);
6302 SDValue ShAmt = Op.getOperand(2);
6303 SDValue ARMcc;
6304 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6305 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
6306
6307 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
6308
6309 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6310 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6311 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
6312 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6313 DAG.getConstant(VTBits, dl, MVT::i32));
6314 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
6315 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6316 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6317 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6318 ISD::SETGE, ARMcc, DAG, dl);
6319 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift,
6320 ARMcc, CCR, CmpLo);
6321
6322 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
6323 SDValue HiBigShift = Opc == ISD::SRA
6324 ? DAG.getNode(Opc, dl, VT, ShOpHi,
6325 DAG.getConstant(VTBits - 1, dl, VT))
6326 : DAG.getConstant(0, dl, VT);
6327 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6328 ISD::SETGE, ARMcc, DAG, dl);
6329 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
6330 ARMcc, CCR, CmpHi);
6331
6332 SDValue Ops[2] = { Lo, Hi };
6333 return DAG.getMergeValues(Ops, dl);
6334 }
6335
6336 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
6337 /// i32 values and take a 2 x i32 value to shift plus a shift amount.
LowerShiftLeftParts(SDValue Op,SelectionDAG & DAG) const6338 SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
6339 SelectionDAG &DAG) const {
6340 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6341 EVT VT = Op.getValueType();
6342 unsigned VTBits = VT.getSizeInBits();
6343 SDLoc dl(Op);
6344 SDValue ShOpLo = Op.getOperand(0);
6345 SDValue ShOpHi = Op.getOperand(1);
6346 SDValue ShAmt = Op.getOperand(2);
6347 SDValue ARMcc;
6348 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6349
6350 assert(Op.getOpcode() == ISD::SHL_PARTS);
6351 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6352 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6353 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
6354 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
6355 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6356
6357 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6358 DAG.getConstant(VTBits, dl, MVT::i32));
6359 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
6360 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6361 ISD::SETGE, ARMcc, DAG, dl);
6362 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
6363 ARMcc, CCR, CmpHi);
6364
6365 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6366 ISD::SETGE, ARMcc, DAG, dl);
6367 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
6368 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
6369 DAG.getConstant(0, dl, VT), ARMcc, CCR, CmpLo);
6370
6371 SDValue Ops[2] = { Lo, Hi };
6372 return DAG.getMergeValues(Ops, dl);
6373 }
6374
LowerGET_ROUNDING(SDValue Op,SelectionDAG & DAG) const6375 SDValue ARMTargetLowering::LowerGET_ROUNDING(SDValue Op,
6376 SelectionDAG &DAG) const {
6377 // The rounding mode is in bits 23:22 of the FPSCR.
6378 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
6379 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
6380 // so that the shift + and get folded into a bitfield extract.
6381 SDLoc dl(Op);
6382 SDValue Chain = Op.getOperand(0);
6383 SDValue Ops[] = {Chain,
6384 DAG.getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6385
6386 SDValue FPSCR =
6387 DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, {MVT::i32, MVT::Other}, Ops);
6388 Chain = FPSCR.getValue(1);
6389 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
6390 DAG.getConstant(1U << 22, dl, MVT::i32));
6391 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
6392 DAG.getConstant(22, dl, MVT::i32));
6393 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
6394 DAG.getConstant(3, dl, MVT::i32));
6395 return DAG.getMergeValues({And, Chain}, dl);
6396 }
6397
LowerSET_ROUNDING(SDValue Op,SelectionDAG & DAG) const6398 SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op,
6399 SelectionDAG &DAG) const {
6400 SDLoc DL(Op);
6401 SDValue Chain = Op->getOperand(0);
6402 SDValue RMValue = Op->getOperand(1);
6403
6404 // The rounding mode is in bits 23:22 of the FPSCR.
6405 // The llvm.set.rounding argument value to ARM rounding mode value mapping
6406 // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
6407 // ((arg - 1) & 3) << 22).
6408 //
6409 // It is expected that the argument of llvm.set.rounding is within the
6410 // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is
6411 // responsibility of the code generated llvm.set.rounding to ensure this
6412 // condition.
6413
6414 // Calculate new value of FPSCR[23:22].
6415 RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
6416 DAG.getConstant(1, DL, MVT::i32));
6417 RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
6418 DAG.getConstant(0x3, DL, MVT::i32));
6419 RMValue = DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
6420 DAG.getConstant(ARM::RoundingBitsPos, DL, MVT::i32));
6421
6422 // Get current value of FPSCR.
6423 SDValue Ops[] = {Chain,
6424 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6425 SDValue FPSCR =
6426 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6427 Chain = FPSCR.getValue(1);
6428 FPSCR = FPSCR.getValue(0);
6429
6430 // Put new rounding mode into FPSCR[23:22].
6431 const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos);
6432 FPSCR = DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
6433 DAG.getConstant(RMMask, DL, MVT::i32));
6434 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCR, RMValue);
6435 SDValue Ops2[] = {
6436 Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
6437 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6438 }
6439
LowerCTTZ(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)6440 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
6441 const ARMSubtarget *ST) {
6442 SDLoc dl(N);
6443 EVT VT = N->getValueType(0);
6444 if (VT.isVector() && ST->hasNEON()) {
6445
6446 // Compute the least significant set bit: LSB = X & -X
6447 SDValue X = N->getOperand(0);
6448 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
6449 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
6450
6451 EVT ElemTy = VT.getVectorElementType();
6452
6453 if (ElemTy == MVT::i8) {
6454 // Compute with: cttz(x) = ctpop(lsb - 1)
6455 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6456 DAG.getTargetConstant(1, dl, ElemTy));
6457 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6458 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6459 }
6460
6461 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6462 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) {
6463 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
6464 unsigned NumBits = ElemTy.getSizeInBits();
6465 SDValue WidthMinus1 =
6466 DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6467 DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
6468 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
6469 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
6470 }
6471
6472 // Compute with: cttz(x) = ctpop(lsb - 1)
6473
6474 // Compute LSB - 1.
6475 SDValue Bits;
6476 if (ElemTy == MVT::i64) {
6477 // Load constant 0xffff'ffff'ffff'ffff to register.
6478 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6479 DAG.getTargetConstant(0x1eff, dl, MVT::i32));
6480 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
6481 } else {
6482 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6483 DAG.getTargetConstant(1, dl, ElemTy));
6484 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6485 }
6486 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6487 }
6488
6489 if (!ST->hasV6T2Ops())
6490 return SDValue();
6491
6492 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
6493 return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
6494 }
6495
LowerCTPOP(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)6496 static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
6497 const ARMSubtarget *ST) {
6498 EVT VT = N->getValueType(0);
6499 SDLoc DL(N);
6500
6501 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
6502 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6503 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6504 "Unexpected type for custom ctpop lowering");
6505
6506 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6507 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6508 SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0));
6509 Res = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Res);
6510
6511 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6512 unsigned EltSize = 8;
6513 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6514 while (EltSize != VT.getScalarSizeInBits()) {
6515 SmallVector<SDValue, 8> Ops;
6516 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddlu, DL,
6517 TLI.getPointerTy(DAG.getDataLayout())));
6518 Ops.push_back(Res);
6519
6520 EltSize *= 2;
6521 NumElts /= 2;
6522 MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
6523 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WidenVT, Ops);
6524 }
6525
6526 return Res;
6527 }
6528
6529 /// Getvshiftimm - Check if this is a valid build_vector for the immediate
6530 /// operand of a vector shift operation, where all the elements of the
6531 /// build_vector must have the same constant integer value.
getVShiftImm(SDValue Op,unsigned ElementBits,int64_t & Cnt)6532 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6533 // Ignore bit_converts.
6534 while (Op.getOpcode() == ISD::BITCAST)
6535 Op = Op.getOperand(0);
6536 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
6537 APInt SplatBits, SplatUndef;
6538 unsigned SplatBitSize;
6539 bool HasAnyUndefs;
6540 if (!BVN ||
6541 !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6542 ElementBits) ||
6543 SplatBitSize > ElementBits)
6544 return false;
6545 Cnt = SplatBits.getSExtValue();
6546 return true;
6547 }
6548
6549 /// isVShiftLImm - Check if this is a valid build_vector for the immediate
6550 /// operand of a vector shift left operation. That value must be in the range:
6551 /// 0 <= Value < ElementBits for a left shift; or
6552 /// 0 <= Value <= ElementBits for a long left shift.
isVShiftLImm(SDValue Op,EVT VT,bool isLong,int64_t & Cnt)6553 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6554 assert(VT.isVector() && "vector shift count is not a vector type");
6555 int64_t ElementBits = VT.getScalarSizeInBits();
6556 if (!getVShiftImm(Op, ElementBits, Cnt))
6557 return false;
6558 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6559 }
6560
6561 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
6562 /// operand of a vector shift right operation. For a shift opcode, the value
6563 /// is positive, but for an intrinsic the value count must be negative. The
6564 /// absolute value must be in the range:
6565 /// 1 <= |Value| <= ElementBits for a right shift; or
6566 /// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
isVShiftRImm(SDValue Op,EVT VT,bool isNarrow,bool isIntrinsic,int64_t & Cnt)6567 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6568 int64_t &Cnt) {
6569 assert(VT.isVector() && "vector shift count is not a vector type");
6570 int64_t ElementBits = VT.getScalarSizeInBits();
6571 if (!getVShiftImm(Op, ElementBits, Cnt))
6572 return false;
6573 if (!isIntrinsic)
6574 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6575 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6576 Cnt = -Cnt;
6577 return true;
6578 }
6579 return false;
6580 }
6581
LowerShift(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)6582 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6583 const ARMSubtarget *ST) {
6584 EVT VT = N->getValueType(0);
6585 SDLoc dl(N);
6586 int64_t Cnt;
6587
6588 if (!VT.isVector())
6589 return SDValue();
6590
6591 // We essentially have two forms here. Shift by an immediate and shift by a
6592 // vector register (there are also shift by a gpr, but that is just handled
6593 // with a tablegen pattern). We cannot easily match shift by an immediate in
6594 // tablegen so we do that here and generate a VSHLIMM/VSHRsIMM/VSHRuIMM.
6595 // For shifting by a vector, we don't have VSHR, only VSHL (which can be
6596 // signed or unsigned, and a negative shift indicates a shift right).
6597 if (N->getOpcode() == ISD::SHL) {
6598 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt))
6599 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
6600 DAG.getConstant(Cnt, dl, MVT::i32));
6601 return DAG.getNode(ARMISD::VSHLu, dl, VT, N->getOperand(0),
6602 N->getOperand(1));
6603 }
6604
6605 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
6606 "unexpected vector shift opcode");
6607
6608 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
6609 unsigned VShiftOpc =
6610 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
6611 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
6612 DAG.getConstant(Cnt, dl, MVT::i32));
6613 }
6614
6615 // Other right shifts we don't have operations for (we use a shift left by a
6616 // negative number).
6617 EVT ShiftVT = N->getOperand(1).getValueType();
6618 SDValue NegatedCount = DAG.getNode(
6619 ISD::SUB, dl, ShiftVT, getZeroVector(ShiftVT, DAG, dl), N->getOperand(1));
6620 unsigned VShiftOpc =
6621 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
6622 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), NegatedCount);
6623 }
6624
Expand64BitShift(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)6625 static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6626 const ARMSubtarget *ST) {
6627 EVT VT = N->getValueType(0);
6628 SDLoc dl(N);
6629
6630 // We can get here for a node like i32 = ISD::SHL i32, i64
6631 if (VT != MVT::i64)
6632 return SDValue();
6633
6634 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||
6635 N->getOpcode() == ISD::SHL) &&
6636 "Unknown shift to lower!");
6637
6638 unsigned ShOpc = N->getOpcode();
6639 if (ST->hasMVEIntegerOps()) {
6640 SDValue ShAmt = N->getOperand(1);
6641 unsigned ShPartsOpc = ARMISD::LSLL;
6642 ConstantSDNode *Con = dyn_cast<ConstantSDNode>(ShAmt);
6643
6644 // If the shift amount is greater than 32 or has a greater bitwidth than 64
6645 // then do the default optimisation
6646 if (ShAmt->getValueType(0).getSizeInBits() > 64 ||
6647 (Con && (Con->getZExtValue() == 0 || Con->getZExtValue() >= 32)))
6648 return SDValue();
6649
6650 // Extract the lower 32 bits of the shift amount if it's not an i32
6651 if (ShAmt->getValueType(0) != MVT::i32)
6652 ShAmt = DAG.getZExtOrTrunc(ShAmt, dl, MVT::i32);
6653
6654 if (ShOpc == ISD::SRL) {
6655 if (!Con)
6656 // There is no t2LSRLr instruction so negate and perform an lsll if the
6657 // shift amount is in a register, emulating a right shift.
6658 ShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6659 DAG.getConstant(0, dl, MVT::i32), ShAmt);
6660 else
6661 // Else generate an lsrl on the immediate shift amount
6662 ShPartsOpc = ARMISD::LSRL;
6663 } else if (ShOpc == ISD::SRA)
6664 ShPartsOpc = ARMISD::ASRL;
6665
6666 // Lower 32 bits of the destination/source
6667 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6668 DAG.getConstant(0, dl, MVT::i32));
6669 // Upper 32 bits of the destination/source
6670 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6671 DAG.getConstant(1, dl, MVT::i32));
6672
6673 // Generate the shift operation as computed above
6674 Lo = DAG.getNode(ShPartsOpc, dl, DAG.getVTList(MVT::i32, MVT::i32), Lo, Hi,
6675 ShAmt);
6676 // The upper 32 bits come from the second return value of lsll
6677 Hi = SDValue(Lo.getNode(), 1);
6678 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6679 }
6680
6681 // We only lower SRA, SRL of 1 here, all others use generic lowering.
6682 if (!isOneConstant(N->getOperand(1)) || N->getOpcode() == ISD::SHL)
6683 return SDValue();
6684
6685 // If we are in thumb mode, we don't have RRX.
6686 if (ST->isThumb1Only())
6687 return SDValue();
6688
6689 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
6690 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6691 DAG.getConstant(0, dl, MVT::i32));
6692 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6693 DAG.getConstant(1, dl, MVT::i32));
6694
6695 // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and
6696 // captures the result into a carry flag.
6697 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG;
6698 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, MVT::Glue), Hi);
6699
6700 // The low part is an ARMISD::RRX operand, which shifts the carry in.
6701 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
6702
6703 // Merge the pieces into a single i64 value.
6704 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6705 }
6706
LowerVSETCC(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)6707 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6708 const ARMSubtarget *ST) {
6709 bool Invert = false;
6710 bool Swap = false;
6711 unsigned Opc = ARMCC::AL;
6712
6713 SDValue Op0 = Op.getOperand(0);
6714 SDValue Op1 = Op.getOperand(1);
6715 SDValue CC = Op.getOperand(2);
6716 EVT VT = Op.getValueType();
6717 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
6718 SDLoc dl(Op);
6719
6720 EVT CmpVT;
6721 if (ST->hasNEON())
6722 CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
6723 else {
6724 assert(ST->hasMVEIntegerOps() &&
6725 "No hardware support for integer vector comparison!");
6726
6727 if (Op.getValueType().getVectorElementType() != MVT::i1)
6728 return SDValue();
6729
6730 // Make sure we expand floating point setcc to scalar if we do not have
6731 // mve.fp, so that we can handle them from there.
6732 if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps())
6733 return SDValue();
6734
6735 CmpVT = VT;
6736 }
6737
6738 if (Op0.getValueType().getVectorElementType() == MVT::i64 &&
6739 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) {
6740 // Special-case integer 64-bit equality comparisons. They aren't legal,
6741 // but they can be lowered with a few vector instructions.
6742 unsigned CmpElements = CmpVT.getVectorNumElements() * 2;
6743 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, CmpElements);
6744 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0);
6745 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1);
6746 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1,
6747 DAG.getCondCode(ISD::SETEQ));
6748 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
6749 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed);
6750 Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged);
6751 if (SetCCOpcode == ISD::SETNE)
6752 Merged = DAG.getNOT(dl, Merged, CmpVT);
6753 Merged = DAG.getSExtOrTrunc(Merged, dl, VT);
6754 return Merged;
6755 }
6756
6757 if (CmpVT.getVectorElementType() == MVT::i64)
6758 // 64-bit comparisons are not legal in general.
6759 return SDValue();
6760
6761 if (Op1.getValueType().isFloatingPoint()) {
6762 switch (SetCCOpcode) {
6763 default: llvm_unreachable("Illegal FP comparison");
6764 case ISD::SETUNE:
6765 case ISD::SETNE:
6766 if (ST->hasMVEFloatOps()) {
6767 Opc = ARMCC::NE; break;
6768 } else {
6769 Invert = true; [[fallthrough]];
6770 }
6771 case ISD::SETOEQ:
6772 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6773 case ISD::SETOLT:
6774 case ISD::SETLT: Swap = true; [[fallthrough]];
6775 case ISD::SETOGT:
6776 case ISD::SETGT: Opc = ARMCC::GT; break;
6777 case ISD::SETOLE:
6778 case ISD::SETLE: Swap = true; [[fallthrough]];
6779 case ISD::SETOGE:
6780 case ISD::SETGE: Opc = ARMCC::GE; break;
6781 case ISD::SETUGE: Swap = true; [[fallthrough]];
6782 case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break;
6783 case ISD::SETUGT: Swap = true; [[fallthrough]];
6784 case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break;
6785 case ISD::SETUEQ: Invert = true; [[fallthrough]];
6786 case ISD::SETONE: {
6787 // Expand this to (OLT | OGT).
6788 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6789 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6790 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6791 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6792 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6793 if (Invert)
6794 Result = DAG.getNOT(dl, Result, VT);
6795 return Result;
6796 }
6797 case ISD::SETUO: Invert = true; [[fallthrough]];
6798 case ISD::SETO: {
6799 // Expand this to (OLT | OGE).
6800 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6801 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6802 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6803 DAG.getConstant(ARMCC::GE, dl, MVT::i32));
6804 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6805 if (Invert)
6806 Result = DAG.getNOT(dl, Result, VT);
6807 return Result;
6808 }
6809 }
6810 } else {
6811 // Integer comparisons.
6812 switch (SetCCOpcode) {
6813 default: llvm_unreachable("Illegal integer comparison");
6814 case ISD::SETNE:
6815 if (ST->hasMVEIntegerOps()) {
6816 Opc = ARMCC::NE; break;
6817 } else {
6818 Invert = true; [[fallthrough]];
6819 }
6820 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6821 case ISD::SETLT: Swap = true; [[fallthrough]];
6822 case ISD::SETGT: Opc = ARMCC::GT; break;
6823 case ISD::SETLE: Swap = true; [[fallthrough]];
6824 case ISD::SETGE: Opc = ARMCC::GE; break;
6825 case ISD::SETULT: Swap = true; [[fallthrough]];
6826 case ISD::SETUGT: Opc = ARMCC::HI; break;
6827 case ISD::SETULE: Swap = true; [[fallthrough]];
6828 case ISD::SETUGE: Opc = ARMCC::HS; break;
6829 }
6830
6831 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
6832 if (ST->hasNEON() && Opc == ARMCC::EQ) {
6833 SDValue AndOp;
6834 if (ISD::isBuildVectorAllZeros(Op1.getNode()))
6835 AndOp = Op0;
6836 else if (ISD::isBuildVectorAllZeros(Op0.getNode()))
6837 AndOp = Op1;
6838
6839 // Ignore bitconvert.
6840 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
6841 AndOp = AndOp.getOperand(0);
6842
6843 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
6844 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
6845 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
6846 SDValue Result = DAG.getNode(ARMISD::VTST, dl, CmpVT, Op0, Op1);
6847 if (!Invert)
6848 Result = DAG.getNOT(dl, Result, VT);
6849 return Result;
6850 }
6851 }
6852 }
6853
6854 if (Swap)
6855 std::swap(Op0, Op1);
6856
6857 // If one of the operands is a constant vector zero, attempt to fold the
6858 // comparison to a specialized compare-against-zero form.
6859 if (ISD::isBuildVectorAllZeros(Op0.getNode()) &&
6860 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::EQ ||
6861 Opc == ARMCC::NE)) {
6862 if (Opc == ARMCC::GE)
6863 Opc = ARMCC::LE;
6864 else if (Opc == ARMCC::GT)
6865 Opc = ARMCC::LT;
6866 std::swap(Op0, Op1);
6867 }
6868
6869 SDValue Result;
6870 if (ISD::isBuildVectorAllZeros(Op1.getNode()) &&
6871 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::LE ||
6872 Opc == ARMCC::LT || Opc == ARMCC::NE || Opc == ARMCC::EQ))
6873 Result = DAG.getNode(ARMISD::VCMPZ, dl, CmpVT, Op0,
6874 DAG.getConstant(Opc, dl, MVT::i32));
6875 else
6876 Result = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6877 DAG.getConstant(Opc, dl, MVT::i32));
6878
6879 Result = DAG.getSExtOrTrunc(Result, dl, VT);
6880
6881 if (Invert)
6882 Result = DAG.getNOT(dl, Result, VT);
6883
6884 return Result;
6885 }
6886
LowerSETCCCARRY(SDValue Op,SelectionDAG & DAG)6887 static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6888 SDValue LHS = Op.getOperand(0);
6889 SDValue RHS = Op.getOperand(1);
6890 SDValue Carry = Op.getOperand(2);
6891 SDValue Cond = Op.getOperand(3);
6892 SDLoc DL(Op);
6893
6894 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
6895
6896 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we
6897 // have to invert the carry first.
6898 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
6899 DAG.getConstant(1, DL, MVT::i32), Carry);
6900 // This converts the boolean value carry into the carry flag.
6901 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
6902
6903 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
6904 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
6905
6906 SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
6907 SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
6908 SDValue ARMcc = DAG.getConstant(
6909 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32);
6910 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6911 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR,
6912 Cmp.getValue(1), SDValue());
6913 return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
6914 CCR, Chain.getValue(1));
6915 }
6916
6917 /// isVMOVModifiedImm - Check if the specified splat value corresponds to a
6918 /// valid vector constant for a NEON or MVE instruction with a "modified
6919 /// immediate" operand (e.g., VMOV). If so, return the encoded value.
isVMOVModifiedImm(uint64_t SplatBits,uint64_t SplatUndef,unsigned SplatBitSize,SelectionDAG & DAG,const SDLoc & dl,EVT & VT,EVT VectorVT,VMOVModImmType type)6920 static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6921 unsigned SplatBitSize, SelectionDAG &DAG,
6922 const SDLoc &dl, EVT &VT, EVT VectorVT,
6923 VMOVModImmType type) {
6924 unsigned OpCmode, Imm;
6925 bool is128Bits = VectorVT.is128BitVector();
6926
6927 // SplatBitSize is set to the smallest size that splats the vector, so a
6928 // zero vector will always have SplatBitSize == 8. However, NEON modified
6929 // immediate instructions others than VMOV do not support the 8-bit encoding
6930 // of a zero vector, and the default encoding of zero is supposed to be the
6931 // 32-bit version.
6932 if (SplatBits == 0)
6933 SplatBitSize = 32;
6934
6935 switch (SplatBitSize) {
6936 case 8:
6937 if (type != VMOVModImm)
6938 return SDValue();
6939 // Any 1-byte value is OK. Op=0, Cmode=1110.
6940 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
6941 OpCmode = 0xe;
6942 Imm = SplatBits;
6943 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6944 break;
6945
6946 case 16:
6947 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
6948 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6949 if ((SplatBits & ~0xff) == 0) {
6950 // Value = 0x00nn: Op=x, Cmode=100x.
6951 OpCmode = 0x8;
6952 Imm = SplatBits;
6953 break;
6954 }
6955 if ((SplatBits & ~0xff00) == 0) {
6956 // Value = 0xnn00: Op=x, Cmode=101x.
6957 OpCmode = 0xa;
6958 Imm = SplatBits >> 8;
6959 break;
6960 }
6961 return SDValue();
6962
6963 case 32:
6964 // NEON's 32-bit VMOV supports splat values where:
6965 // * only one byte is nonzero, or
6966 // * the least significant byte is 0xff and the second byte is nonzero, or
6967 // * the least significant 2 bytes are 0xff and the third is nonzero.
6968 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
6969 if ((SplatBits & ~0xff) == 0) {
6970 // Value = 0x000000nn: Op=x, Cmode=000x.
6971 OpCmode = 0;
6972 Imm = SplatBits;
6973 break;
6974 }
6975 if ((SplatBits & ~0xff00) == 0) {
6976 // Value = 0x0000nn00: Op=x, Cmode=001x.
6977 OpCmode = 0x2;
6978 Imm = SplatBits >> 8;
6979 break;
6980 }
6981 if ((SplatBits & ~0xff0000) == 0) {
6982 // Value = 0x00nn0000: Op=x, Cmode=010x.
6983 OpCmode = 0x4;
6984 Imm = SplatBits >> 16;
6985 break;
6986 }
6987 if ((SplatBits & ~0xff000000) == 0) {
6988 // Value = 0xnn000000: Op=x, Cmode=011x.
6989 OpCmode = 0x6;
6990 Imm = SplatBits >> 24;
6991 break;
6992 }
6993
6994 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
6995 if (type == OtherModImm) return SDValue();
6996
6997 if ((SplatBits & ~0xffff) == 0 &&
6998 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
6999 // Value = 0x0000nnff: Op=x, Cmode=1100.
7000 OpCmode = 0xc;
7001 Imm = SplatBits >> 8;
7002 break;
7003 }
7004
7005 // cmode == 0b1101 is not supported for MVE VMVN
7006 if (type == MVEVMVNModImm)
7007 return SDValue();
7008
7009 if ((SplatBits & ~0xffffff) == 0 &&
7010 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7011 // Value = 0x00nnffff: Op=x, Cmode=1101.
7012 OpCmode = 0xd;
7013 Imm = SplatBits >> 16;
7014 break;
7015 }
7016
7017 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
7018 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
7019 // VMOV.I32. A (very) minor optimization would be to replicate the value
7020 // and fall through here to test for a valid 64-bit splat. But, then the
7021 // caller would also need to check and handle the change in size.
7022 return SDValue();
7023
7024 case 64: {
7025 if (type != VMOVModImm)
7026 return SDValue();
7027 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
7028 uint64_t BitMask = 0xff;
7029 unsigned ImmMask = 1;
7030 Imm = 0;
7031 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7032 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7033 Imm |= ImmMask;
7034 } else if ((SplatBits & BitMask) != 0) {
7035 return SDValue();
7036 }
7037 BitMask <<= 8;
7038 ImmMask <<= 1;
7039 }
7040
7041 if (DAG.getDataLayout().isBigEndian()) {
7042 // Reverse the order of elements within the vector.
7043 unsigned BytesPerElem = VectorVT.getScalarSizeInBits() / 8;
7044 unsigned Mask = (1 << BytesPerElem) - 1;
7045 unsigned NumElems = 8 / BytesPerElem;
7046 unsigned NewImm = 0;
7047 for (unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
7048 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
7049 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
7050 }
7051 Imm = NewImm;
7052 }
7053
7054 // Op=1, Cmode=1110.
7055 OpCmode = 0x1e;
7056 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7057 break;
7058 }
7059
7060 default:
7061 llvm_unreachable("unexpected size for isVMOVModifiedImm");
7062 }
7063
7064 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode, Imm);
7065 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
7066 }
7067
LowerConstantFP(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST) const7068 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
7069 const ARMSubtarget *ST) const {
7070 EVT VT = Op.getValueType();
7071 bool IsDouble = (VT == MVT::f64);
7072 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op);
7073 const APFloat &FPVal = CFP->getValueAPF();
7074
7075 // Prevent floating-point constants from using literal loads
7076 // when execute-only is enabled.
7077 if (ST->genExecuteOnly()) {
7078 // If we can represent the constant as an immediate, don't lower it
7079 if (isFPImmLegal(FPVal, VT))
7080 return Op;
7081 // Otherwise, construct as integer, and move to float register
7082 APInt INTVal = FPVal.bitcastToAPInt();
7083 SDLoc DL(CFP);
7084 switch (VT.getSimpleVT().SimpleTy) {
7085 default:
7086 llvm_unreachable("Unknown floating point type!");
7087 break;
7088 case MVT::f64: {
7089 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
7090 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
7091 return DAG.getNode(ARMISD::VMOVDRR, DL, MVT::f64, Lo, Hi);
7092 }
7093 case MVT::f32:
7094 return DAG.getNode(ARMISD::VMOVSR, DL, VT,
7095 DAG.getConstant(INTVal, DL, MVT::i32));
7096 }
7097 }
7098
7099 if (!ST->hasVFP3Base())
7100 return SDValue();
7101
7102 // Use the default (constant pool) lowering for double constants when we have
7103 // an SP-only FPU
7104 if (IsDouble && !Subtarget->hasFP64())
7105 return SDValue();
7106
7107 // Try splatting with a VMOV.f32...
7108 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal);
7109
7110 if (ImmVal != -1) {
7111 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
7112 // We have code in place to select a valid ConstantFP already, no need to
7113 // do any mangling.
7114 return Op;
7115 }
7116
7117 // It's a float and we are trying to use NEON operations where
7118 // possible. Lower it to a splat followed by an extract.
7119 SDLoc DL(Op);
7120 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
7121 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
7122 NewVal);
7123 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
7124 DAG.getConstant(0, DL, MVT::i32));
7125 }
7126
7127 // The rest of our options are NEON only, make sure that's allowed before
7128 // proceeding..
7129 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
7130 return SDValue();
7131
7132 EVT VMovVT;
7133 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
7134
7135 // It wouldn't really be worth bothering for doubles except for one very
7136 // important value, which does happen to match: 0.0. So make sure we don't do
7137 // anything stupid.
7138 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7139 return SDValue();
7140
7141 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
7142 SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
7143 VMovVT, VT, VMOVModImm);
7144 if (NewVal != SDValue()) {
7145 SDLoc DL(Op);
7146 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
7147 NewVal);
7148 if (IsDouble)
7149 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
7150
7151 // It's a float: cast and extract a vector element.
7152 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
7153 VecConstant);
7154 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
7155 DAG.getConstant(0, DL, MVT::i32));
7156 }
7157
7158 // Finally, try a VMVN.i32
7159 NewVal = isVMOVModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
7160 VT, VMVNModImm);
7161 if (NewVal != SDValue()) {
7162 SDLoc DL(Op);
7163 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
7164
7165 if (IsDouble)
7166 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
7167
7168 // It's a float: cast and extract a vector element.
7169 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
7170 VecConstant);
7171 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
7172 DAG.getConstant(0, DL, MVT::i32));
7173 }
7174
7175 return SDValue();
7176 }
7177
7178 // check if an VEXT instruction can handle the shuffle mask when the
7179 // vector sources of the shuffle are the same.
isSingletonVEXTMask(ArrayRef<int> M,EVT VT,unsigned & Imm)7180 static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
7181 unsigned NumElts = VT.getVectorNumElements();
7182
7183 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7184 if (M[0] < 0)
7185 return false;
7186
7187 Imm = M[0];
7188
7189 // If this is a VEXT shuffle, the immediate value is the index of the first
7190 // element. The other shuffle indices must be the successive elements after
7191 // the first one.
7192 unsigned ExpectedElt = Imm;
7193 for (unsigned i = 1; i < NumElts; ++i) {
7194 // Increment the expected index. If it wraps around, just follow it
7195 // back to index zero and keep going.
7196 ++ExpectedElt;
7197 if (ExpectedElt == NumElts)
7198 ExpectedElt = 0;
7199
7200 if (M[i] < 0) continue; // ignore UNDEF indices
7201 if (ExpectedElt != static_cast<unsigned>(M[i]))
7202 return false;
7203 }
7204
7205 return true;
7206 }
7207
isVEXTMask(ArrayRef<int> M,EVT VT,bool & ReverseVEXT,unsigned & Imm)7208 static bool isVEXTMask(ArrayRef<int> M, EVT VT,
7209 bool &ReverseVEXT, unsigned &Imm) {
7210 unsigned NumElts = VT.getVectorNumElements();
7211 ReverseVEXT = false;
7212
7213 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7214 if (M[0] < 0)
7215 return false;
7216
7217 Imm = M[0];
7218
7219 // If this is a VEXT shuffle, the immediate value is the index of the first
7220 // element. The other shuffle indices must be the successive elements after
7221 // the first one.
7222 unsigned ExpectedElt = Imm;
7223 for (unsigned i = 1; i < NumElts; ++i) {
7224 // Increment the expected index. If it wraps around, it may still be
7225 // a VEXT but the source vectors must be swapped.
7226 ExpectedElt += 1;
7227 if (ExpectedElt == NumElts * 2) {
7228 ExpectedElt = 0;
7229 ReverseVEXT = true;
7230 }
7231
7232 if (M[i] < 0) continue; // ignore UNDEF indices
7233 if (ExpectedElt != static_cast<unsigned>(M[i]))
7234 return false;
7235 }
7236
7237 // Adjust the index value if the source operands will be swapped.
7238 if (ReverseVEXT)
7239 Imm -= NumElts;
7240
7241 return true;
7242 }
7243
isVTBLMask(ArrayRef<int> M,EVT VT)7244 static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
7245 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
7246 // range, then 0 is placed into the resulting vector. So pretty much any mask
7247 // of 8 elements can work here.
7248 return VT == MVT::v8i8 && M.size() == 8;
7249 }
7250
SelectPairHalf(unsigned Elements,ArrayRef<int> Mask,unsigned Index)7251 static unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask,
7252 unsigned Index) {
7253 if (Mask.size() == Elements * 2)
7254 return Index / Elements;
7255 return Mask[Index] == 0 ? 0 : 1;
7256 }
7257
7258 // Checks whether the shuffle mask represents a vector transpose (VTRN) by
7259 // checking that pairs of elements in the shuffle mask represent the same index
7260 // in each vector, incrementing the expected index by 2 at each step.
7261 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
7262 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
7263 // v2={e,f,g,h}
7264 // WhichResult gives the offset for each element in the mask based on which
7265 // of the two results it belongs to.
7266 //
7267 // The transpose can be represented either as:
7268 // result1 = shufflevector v1, v2, result1_shuffle_mask
7269 // result2 = shufflevector v1, v2, result2_shuffle_mask
7270 // where v1/v2 and the shuffle masks have the same number of elements
7271 // (here WhichResult (see below) indicates which result is being checked)
7272 //
7273 // or as:
7274 // results = shufflevector v1, v2, shuffle_mask
7275 // where both results are returned in one vector and the shuffle mask has twice
7276 // as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
7277 // want to check the low half and high half of the shuffle mask as if it were
7278 // the other case
isVTRNMask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)7279 static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7280 unsigned EltSz = VT.getScalarSizeInBits();
7281 if (EltSz == 64)
7282 return false;
7283
7284 unsigned NumElts = VT.getVectorNumElements();
7285 if (M.size() != NumElts && M.size() != NumElts*2)
7286 return false;
7287
7288 // If the mask is twice as long as the input vector then we need to check the
7289 // upper and lower parts of the mask with a matching value for WhichResult
7290 // FIXME: A mask with only even values will be rejected in case the first
7291 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
7292 // M[0] is used to determine WhichResult
7293 for (unsigned i = 0; i < M.size(); i += NumElts) {
7294 WhichResult = SelectPairHalf(NumElts, M, i);
7295 for (unsigned j = 0; j < NumElts; j += 2) {
7296 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
7297 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
7298 return false;
7299 }
7300 }
7301
7302 if (M.size() == NumElts*2)
7303 WhichResult = 0;
7304
7305 return true;
7306 }
7307
7308 /// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
7309 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7310 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
isVTRN_v_undef_Mask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)7311 static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7312 unsigned EltSz = VT.getScalarSizeInBits();
7313 if (EltSz == 64)
7314 return false;
7315
7316 unsigned NumElts = VT.getVectorNumElements();
7317 if (M.size() != NumElts && M.size() != NumElts*2)
7318 return false;
7319
7320 for (unsigned i = 0; i < M.size(); i += NumElts) {
7321 WhichResult = SelectPairHalf(NumElts, M, i);
7322 for (unsigned j = 0; j < NumElts; j += 2) {
7323 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
7324 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
7325 return false;
7326 }
7327 }
7328
7329 if (M.size() == NumElts*2)
7330 WhichResult = 0;
7331
7332 return true;
7333 }
7334
7335 // Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
7336 // that the mask elements are either all even and in steps of size 2 or all odd
7337 // and in steps of size 2.
7338 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
7339 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
7340 // v2={e,f,g,h}
7341 // Requires similar checks to that of isVTRNMask with
7342 // respect the how results are returned.
isVUZPMask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)7343 static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7344 unsigned EltSz = VT.getScalarSizeInBits();
7345 if (EltSz == 64)
7346 return false;
7347
7348 unsigned NumElts = VT.getVectorNumElements();
7349 if (M.size() != NumElts && M.size() != NumElts*2)
7350 return false;
7351
7352 for (unsigned i = 0; i < M.size(); i += NumElts) {
7353 WhichResult = SelectPairHalf(NumElts, M, i);
7354 for (unsigned j = 0; j < NumElts; ++j) {
7355 if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
7356 return false;
7357 }
7358 }
7359
7360 if (M.size() == NumElts*2)
7361 WhichResult = 0;
7362
7363 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7364 if (VT.is64BitVector() && EltSz == 32)
7365 return false;
7366
7367 return true;
7368 }
7369
7370 /// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
7371 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7372 /// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
isVUZP_v_undef_Mask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)7373 static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7374 unsigned EltSz = VT.getScalarSizeInBits();
7375 if (EltSz == 64)
7376 return false;
7377
7378 unsigned NumElts = VT.getVectorNumElements();
7379 if (M.size() != NumElts && M.size() != NumElts*2)
7380 return false;
7381
7382 unsigned Half = NumElts / 2;
7383 for (unsigned i = 0; i < M.size(); i += NumElts) {
7384 WhichResult = SelectPairHalf(NumElts, M, i);
7385 for (unsigned j = 0; j < NumElts; j += Half) {
7386 unsigned Idx = WhichResult;
7387 for (unsigned k = 0; k < Half; ++k) {
7388 int MIdx = M[i + j + k];
7389 if (MIdx >= 0 && (unsigned) MIdx != Idx)
7390 return false;
7391 Idx += 2;
7392 }
7393 }
7394 }
7395
7396 if (M.size() == NumElts*2)
7397 WhichResult = 0;
7398
7399 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7400 if (VT.is64BitVector() && EltSz == 32)
7401 return false;
7402
7403 return true;
7404 }
7405
7406 // Checks whether the shuffle mask represents a vector zip (VZIP) by checking
7407 // that pairs of elements of the shufflemask represent the same index in each
7408 // vector incrementing sequentially through the vectors.
7409 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
7410 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
7411 // v2={e,f,g,h}
7412 // Requires similar checks to that of isVTRNMask with respect the how results
7413 // are returned.
isVZIPMask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)7414 static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7415 unsigned EltSz = VT.getScalarSizeInBits();
7416 if (EltSz == 64)
7417 return false;
7418
7419 unsigned NumElts = VT.getVectorNumElements();
7420 if (M.size() != NumElts && M.size() != NumElts*2)
7421 return false;
7422
7423 for (unsigned i = 0; i < M.size(); i += NumElts) {
7424 WhichResult = SelectPairHalf(NumElts, M, i);
7425 unsigned Idx = WhichResult * NumElts / 2;
7426 for (unsigned j = 0; j < NumElts; j += 2) {
7427 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
7428 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
7429 return false;
7430 Idx += 1;
7431 }
7432 }
7433
7434 if (M.size() == NumElts*2)
7435 WhichResult = 0;
7436
7437 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7438 if (VT.is64BitVector() && EltSz == 32)
7439 return false;
7440
7441 return true;
7442 }
7443
7444 /// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
7445 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7446 /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
isVZIP_v_undef_Mask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)7447 static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7448 unsigned EltSz = VT.getScalarSizeInBits();
7449 if (EltSz == 64)
7450 return false;
7451
7452 unsigned NumElts = VT.getVectorNumElements();
7453 if (M.size() != NumElts && M.size() != NumElts*2)
7454 return false;
7455
7456 for (unsigned i = 0; i < M.size(); i += NumElts) {
7457 WhichResult = SelectPairHalf(NumElts, M, i);
7458 unsigned Idx = WhichResult * NumElts / 2;
7459 for (unsigned j = 0; j < NumElts; j += 2) {
7460 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
7461 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
7462 return false;
7463 Idx += 1;
7464 }
7465 }
7466
7467 if (M.size() == NumElts*2)
7468 WhichResult = 0;
7469
7470 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7471 if (VT.is64BitVector() && EltSz == 32)
7472 return false;
7473
7474 return true;
7475 }
7476
7477 /// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
7478 /// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask,EVT VT,unsigned & WhichResult,bool & isV_UNDEF)7479 static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
7480 unsigned &WhichResult,
7481 bool &isV_UNDEF) {
7482 isV_UNDEF = false;
7483 if (isVTRNMask(ShuffleMask, VT, WhichResult))
7484 return ARMISD::VTRN;
7485 if (isVUZPMask(ShuffleMask, VT, WhichResult))
7486 return ARMISD::VUZP;
7487 if (isVZIPMask(ShuffleMask, VT, WhichResult))
7488 return ARMISD::VZIP;
7489
7490 isV_UNDEF = true;
7491 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
7492 return ARMISD::VTRN;
7493 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
7494 return ARMISD::VUZP;
7495 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
7496 return ARMISD::VZIP;
7497
7498 return 0;
7499 }
7500
7501 /// \return true if this is a reverse operation on an vector.
isReverseMask(ArrayRef<int> M,EVT VT)7502 static bool isReverseMask(ArrayRef<int> M, EVT VT) {
7503 unsigned NumElts = VT.getVectorNumElements();
7504 // Make sure the mask has the right size.
7505 if (NumElts != M.size())
7506 return false;
7507
7508 // Look for <15, ..., 3, -1, 1, 0>.
7509 for (unsigned i = 0; i != NumElts; ++i)
7510 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
7511 return false;
7512
7513 return true;
7514 }
7515
isTruncMask(ArrayRef<int> M,EVT VT,bool Top,bool SingleSource)7516 static bool isTruncMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7517 unsigned NumElts = VT.getVectorNumElements();
7518 // Make sure the mask has the right size.
7519 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7520 return false;
7521
7522 // Half-width truncation patterns (e.g. v4i32 -> v8i16):
7523 // !Top && SingleSource: <0, 2, 4, 6, 0, 2, 4, 6>
7524 // !Top && !SingleSource: <0, 2, 4, 6, 8, 10, 12, 14>
7525 // Top && SingleSource: <1, 3, 5, 7, 1, 3, 5, 7>
7526 // Top && !SingleSource: <1, 3, 5, 7, 9, 11, 13, 15>
7527 int Ofs = Top ? 1 : 0;
7528 int Upper = SingleSource ? 0 : NumElts;
7529 for (int i = 0, e = NumElts / 2; i != e; ++i) {
7530 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7531 return false;
7532 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs + Upper)
7533 return false;
7534 }
7535 return true;
7536 }
7537
isVMOVNMask(ArrayRef<int> M,EVT VT,bool Top,bool SingleSource)7538 static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7539 unsigned NumElts = VT.getVectorNumElements();
7540 // Make sure the mask has the right size.
7541 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7542 return false;
7543
7544 // If Top
7545 // Look for <0, N, 2, N+2, 4, N+4, ..>.
7546 // This inserts Input2 into Input1
7547 // else if not Top
7548 // Look for <0, N+1, 2, N+3, 4, N+5, ..>
7549 // This inserts Input1 into Input2
7550 unsigned Offset = Top ? 0 : 1;
7551 unsigned N = SingleSource ? 0 : NumElts;
7552 for (unsigned i = 0; i < NumElts; i += 2) {
7553 if (M[i] >= 0 && M[i] != (int)i)
7554 return false;
7555 if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
7556 return false;
7557 }
7558
7559 return true;
7560 }
7561
isVMOVNTruncMask(ArrayRef<int> M,EVT ToVT,bool rev)7562 static bool isVMOVNTruncMask(ArrayRef<int> M, EVT ToVT, bool rev) {
7563 unsigned NumElts = ToVT.getVectorNumElements();
7564 if (NumElts != M.size())
7565 return false;
7566
7567 // Test if the Trunc can be convertable to a VMOVN with this shuffle. We are
7568 // looking for patterns of:
7569 // !rev: 0 N/2 1 N/2+1 2 N/2+2 ...
7570 // rev: N/2 0 N/2+1 1 N/2+2 2 ...
7571
7572 unsigned Off0 = rev ? NumElts / 2 : 0;
7573 unsigned Off1 = rev ? 0 : NumElts / 2;
7574 for (unsigned i = 0; i < NumElts; i += 2) {
7575 if (M[i] >= 0 && M[i] != (int)(Off0 + i / 2))
7576 return false;
7577 if (M[i + 1] >= 0 && M[i + 1] != (int)(Off1 + i / 2))
7578 return false;
7579 }
7580
7581 return true;
7582 }
7583
7584 // Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
7585 // from a pair of inputs. For example:
7586 // BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7587 // FP_ROUND(EXTRACT_ELT(Y, 0),
7588 // FP_ROUND(EXTRACT_ELT(X, 1),
7589 // FP_ROUND(EXTRACT_ELT(Y, 1), ...)
LowerBuildVectorOfFPTrunc(SDValue BV,SelectionDAG & DAG,const ARMSubtarget * ST)7590 static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
7591 const ARMSubtarget *ST) {
7592 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7593 if (!ST->hasMVEFloatOps())
7594 return SDValue();
7595
7596 SDLoc dl(BV);
7597 EVT VT = BV.getValueType();
7598 if (VT != MVT::v8f16)
7599 return SDValue();
7600
7601 // We are looking for a buildvector of fptrunc elements, where all the
7602 // elements are interleavingly extracted from two sources. Check the first two
7603 // items are valid enough and extract some info from them (they are checked
7604 // properly in the loop below).
7605 if (BV.getOperand(0).getOpcode() != ISD::FP_ROUND ||
7606 BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7607 BV.getOperand(0).getOperand(0).getConstantOperandVal(1) != 0)
7608 return SDValue();
7609 if (BV.getOperand(1).getOpcode() != ISD::FP_ROUND ||
7610 BV.getOperand(1).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7611 BV.getOperand(1).getOperand(0).getConstantOperandVal(1) != 0)
7612 return SDValue();
7613 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7614 SDValue Op1 = BV.getOperand(1).getOperand(0).getOperand(0);
7615 if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
7616 return SDValue();
7617
7618 // Check all the values in the BuildVector line up with our expectations.
7619 for (unsigned i = 1; i < 4; i++) {
7620 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7621 return Trunc.getOpcode() == ISD::FP_ROUND &&
7622 Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7623 Trunc.getOperand(0).getOperand(0) == Op &&
7624 Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
7625 };
7626 if (!Check(BV.getOperand(i * 2 + 0), Op0, i))
7627 return SDValue();
7628 if (!Check(BV.getOperand(i * 2 + 1), Op1, i))
7629 return SDValue();
7630 }
7631
7632 SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0,
7633 DAG.getConstant(0, dl, MVT::i32));
7634 return DAG.getNode(ARMISD::VCVTN, dl, VT, N1, Op1,
7635 DAG.getConstant(1, dl, MVT::i32));
7636 }
7637
7638 // Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
7639 // from a single input on alternating lanes. For example:
7640 // BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7641 // FP_ROUND(EXTRACT_ELT(X, 2),
7642 // FP_ROUND(EXTRACT_ELT(X, 4), ...)
LowerBuildVectorOfFPExt(SDValue BV,SelectionDAG & DAG,const ARMSubtarget * ST)7643 static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
7644 const ARMSubtarget *ST) {
7645 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7646 if (!ST->hasMVEFloatOps())
7647 return SDValue();
7648
7649 SDLoc dl(BV);
7650 EVT VT = BV.getValueType();
7651 if (VT != MVT::v4f32)
7652 return SDValue();
7653
7654 // We are looking for a buildvector of fptext elements, where all the
7655 // elements are alternating lanes from a single source. For example <0,2,4,6>
7656 // or <1,3,5,7>. Check the first two items are valid enough and extract some
7657 // info from them (they are checked properly in the loop below).
7658 if (BV.getOperand(0).getOpcode() != ISD::FP_EXTEND ||
7659 BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
7660 return SDValue();
7661 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7662 int Offset = BV.getOperand(0).getOperand(0).getConstantOperandVal(1);
7663 if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
7664 return SDValue();
7665
7666 // Check all the values in the BuildVector line up with our expectations.
7667 for (unsigned i = 1; i < 4; i++) {
7668 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7669 return Trunc.getOpcode() == ISD::FP_EXTEND &&
7670 Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7671 Trunc.getOperand(0).getOperand(0) == Op &&
7672 Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
7673 };
7674 if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
7675 return SDValue();
7676 }
7677
7678 return DAG.getNode(ARMISD::VCVTL, dl, VT, Op0,
7679 DAG.getConstant(Offset, dl, MVT::i32));
7680 }
7681
7682 // If N is an integer constant that can be moved into a register in one
7683 // instruction, return an SDValue of such a constant (will become a MOV
7684 // instruction). Otherwise return null.
IsSingleInstrConstant(SDValue N,SelectionDAG & DAG,const ARMSubtarget * ST,const SDLoc & dl)7685 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7686 const ARMSubtarget *ST, const SDLoc &dl) {
7687 uint64_t Val;
7688 if (!isa<ConstantSDNode>(N))
7689 return SDValue();
7690 Val = cast<ConstantSDNode>(N)->getZExtValue();
7691
7692 if (ST->isThumb1Only()) {
7693 if (Val <= 255 || ~Val <= 255)
7694 return DAG.getConstant(Val, dl, MVT::i32);
7695 } else {
7696 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1)
7697 return DAG.getConstant(Val, dl, MVT::i32);
7698 }
7699 return SDValue();
7700 }
7701
LowerBUILD_VECTOR_i1(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)7702 static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7703 const ARMSubtarget *ST) {
7704 SDLoc dl(Op);
7705 EVT VT = Op.getValueType();
7706
7707 assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!");
7708
7709 unsigned NumElts = VT.getVectorNumElements();
7710 unsigned BoolMask;
7711 unsigned BitsPerBool;
7712 if (NumElts == 2) {
7713 BitsPerBool = 8;
7714 BoolMask = 0xff;
7715 } else if (NumElts == 4) {
7716 BitsPerBool = 4;
7717 BoolMask = 0xf;
7718 } else if (NumElts == 8) {
7719 BitsPerBool = 2;
7720 BoolMask = 0x3;
7721 } else if (NumElts == 16) {
7722 BitsPerBool = 1;
7723 BoolMask = 0x1;
7724 } else
7725 return SDValue();
7726
7727 // If this is a single value copied into all lanes (a splat), we can just sign
7728 // extend that single value
7729 SDValue FirstOp = Op.getOperand(0);
7730 if (!isa<ConstantSDNode>(FirstOp) &&
7731 llvm::all_of(llvm::drop_begin(Op->ops()), [&FirstOp](const SDUse &U) {
7732 return U.get().isUndef() || U.get() == FirstOp;
7733 })) {
7734 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
7735 DAG.getValueType(MVT::i1));
7736 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), Ext);
7737 }
7738
7739 // First create base with bits set where known
7740 unsigned Bits32 = 0;
7741 for (unsigned i = 0; i < NumElts; ++i) {
7742 SDValue V = Op.getOperand(i);
7743 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7744 continue;
7745 bool BitSet = V.isUndef() ? false : cast<ConstantSDNode>(V)->getZExtValue();
7746 if (BitSet)
7747 Bits32 |= BoolMask << (i * BitsPerBool);
7748 }
7749
7750 // Add in unknown nodes
7751 SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
7752 DAG.getConstant(Bits32, dl, MVT::i32));
7753 for (unsigned i = 0; i < NumElts; ++i) {
7754 SDValue V = Op.getOperand(i);
7755 if (isa<ConstantSDNode>(V) || V.isUndef())
7756 continue;
7757 Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V,
7758 DAG.getConstant(i, dl, MVT::i32));
7759 }
7760
7761 return Base;
7762 }
7763
LowerBUILD_VECTORToVIDUP(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)7764 static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
7765 const ARMSubtarget *ST) {
7766 if (!ST->hasMVEIntegerOps())
7767 return SDValue();
7768
7769 // We are looking for a buildvector where each element is Op[0] + i*N
7770 EVT VT = Op.getValueType();
7771 SDValue Op0 = Op.getOperand(0);
7772 unsigned NumElts = VT.getVectorNumElements();
7773
7774 // Get the increment value from operand 1
7775 SDValue Op1 = Op.getOperand(1);
7776 if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(0) != Op0 ||
7777 !isa<ConstantSDNode>(Op1.getOperand(1)))
7778 return SDValue();
7779 unsigned N = Op1.getConstantOperandVal(1);
7780 if (N != 1 && N != 2 && N != 4 && N != 8)
7781 return SDValue();
7782
7783 // Check that each other operand matches
7784 for (unsigned I = 2; I < NumElts; I++) {
7785 SDValue OpI = Op.getOperand(I);
7786 if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(0) != Op0 ||
7787 !isa<ConstantSDNode>(OpI.getOperand(1)) ||
7788 OpI.getConstantOperandVal(1) != I * N)
7789 return SDValue();
7790 }
7791
7792 SDLoc DL(Op);
7793 return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0,
7794 DAG.getConstant(N, DL, MVT::i32));
7795 }
7796
7797 // Returns true if the operation N can be treated as qr instruction variant at
7798 // operand Op.
IsQRMVEInstruction(const SDNode * N,const SDNode * Op)7799 static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) {
7800 switch (N->getOpcode()) {
7801 case ISD::ADD:
7802 case ISD::MUL:
7803 case ISD::SADDSAT:
7804 case ISD::UADDSAT:
7805 return true;
7806 case ISD::SUB:
7807 case ISD::SSUBSAT:
7808 case ISD::USUBSAT:
7809 return N->getOperand(1).getNode() == Op;
7810 case ISD::INTRINSIC_WO_CHAIN:
7811 switch (N->getConstantOperandVal(0)) {
7812 case Intrinsic::arm_mve_add_predicated:
7813 case Intrinsic::arm_mve_mul_predicated:
7814 case Intrinsic::arm_mve_qadd_predicated:
7815 case Intrinsic::arm_mve_vhadd:
7816 case Intrinsic::arm_mve_hadd_predicated:
7817 case Intrinsic::arm_mve_vqdmulh:
7818 case Intrinsic::arm_mve_qdmulh_predicated:
7819 case Intrinsic::arm_mve_vqrdmulh:
7820 case Intrinsic::arm_mve_qrdmulh_predicated:
7821 case Intrinsic::arm_mve_vqdmull:
7822 case Intrinsic::arm_mve_vqdmull_predicated:
7823 return true;
7824 case Intrinsic::arm_mve_sub_predicated:
7825 case Intrinsic::arm_mve_qsub_predicated:
7826 case Intrinsic::arm_mve_vhsub:
7827 case Intrinsic::arm_mve_hsub_predicated:
7828 return N->getOperand(2).getNode() == Op;
7829 default:
7830 return false;
7831 }
7832 default:
7833 return false;
7834 }
7835 }
7836
7837 // If this is a case we can't handle, return null and let the default
7838 // expansion code take care of it.
LowerBUILD_VECTOR(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST) const7839 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7840 const ARMSubtarget *ST) const {
7841 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
7842 SDLoc dl(Op);
7843 EVT VT = Op.getValueType();
7844
7845 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
7846 return LowerBUILD_VECTOR_i1(Op, DAG, ST);
7847
7848 if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
7849 return R;
7850
7851 APInt SplatBits, SplatUndef;
7852 unsigned SplatBitSize;
7853 bool HasAnyUndefs;
7854 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7855 if (SplatUndef.isAllOnes())
7856 return DAG.getUNDEF(VT);
7857
7858 // If all the users of this constant splat are qr instruction variants,
7859 // generate a vdup of the constant.
7860 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize &&
7861 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7862 all_of(BVN->uses(),
7863 [BVN](const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7864 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7865 : SplatBitSize == 16 ? MVT::v8i16
7866 : MVT::v16i8;
7867 SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
7868 SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
7869 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
7870 }
7871
7872 if ((ST->hasNEON() && SplatBitSize <= 64) ||
7873 (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7874 // Check if an immediate VMOV works.
7875 EVT VmovVT;
7876 SDValue Val =
7877 isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
7878 SplatBitSize, DAG, dl, VmovVT, VT, VMOVModImm);
7879
7880 if (Val.getNode()) {
7881 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
7882 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
7883 }
7884
7885 // Try an immediate VMVN.
7886 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7887 Val = isVMOVModifiedImm(
7888 NegatedImm, SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7889 VT, ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
7890 if (Val.getNode()) {
7891 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
7892 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
7893 }
7894
7895 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
7896 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7897 int ImmVal = ARM_AM::getFP32Imm(SplatBits);
7898 if (ImmVal != -1) {
7899 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
7900 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
7901 }
7902 }
7903
7904 // If we are under MVE, generate a VDUP(constant), bitcast to the original
7905 // type.
7906 if (ST->hasMVEIntegerOps() &&
7907 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7908 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7909 : SplatBitSize == 16 ? MVT::v8i16
7910 : MVT::v16i8;
7911 SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
7912 SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
7913 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
7914 }
7915 }
7916 }
7917
7918 // Scan through the operands to see if only one value is used.
7919 //
7920 // As an optimisation, even if more than one value is used it may be more
7921 // profitable to splat with one value then change some lanes.
7922 //
7923 // Heuristically we decide to do this if the vector has a "dominant" value,
7924 // defined as splatted to more than half of the lanes.
7925 unsigned NumElts = VT.getVectorNumElements();
7926 bool isOnlyLowElement = true;
7927 bool usesOnlyOneValue = true;
7928 bool hasDominantValue = false;
7929 bool isConstant = true;
7930
7931 // Map of the number of times a particular SDValue appears in the
7932 // element list.
7933 DenseMap<SDValue, unsigned> ValueCounts;
7934 SDValue Value;
7935 for (unsigned i = 0; i < NumElts; ++i) {
7936 SDValue V = Op.getOperand(i);
7937 if (V.isUndef())
7938 continue;
7939 if (i > 0)
7940 isOnlyLowElement = false;
7941 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
7942 isConstant = false;
7943
7944 ValueCounts.insert(std::make_pair(V, 0));
7945 unsigned &Count = ValueCounts[V];
7946
7947 // Is this value dominant? (takes up more than half of the lanes)
7948 if (++Count > (NumElts / 2)) {
7949 hasDominantValue = true;
7950 Value = V;
7951 }
7952 }
7953 if (ValueCounts.size() != 1)
7954 usesOnlyOneValue = false;
7955 if (!Value.getNode() && !ValueCounts.empty())
7956 Value = ValueCounts.begin()->first;
7957
7958 if (ValueCounts.empty())
7959 return DAG.getUNDEF(VT);
7960
7961 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
7962 // Keep going if we are hitting this case.
7963 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode()))
7964 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
7965
7966 unsigned EltSize = VT.getScalarSizeInBits();
7967
7968 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
7969 // i32 and try again.
7970 if (hasDominantValue && EltSize <= 32) {
7971 if (!isConstant) {
7972 SDValue N;
7973
7974 // If we are VDUPing a value that comes directly from a vector, that will
7975 // cause an unnecessary move to and from a GPR, where instead we could
7976 // just use VDUPLANE. We can only do this if the lane being extracted
7977 // is at a constant index, as the VDUP from lane instructions only have
7978 // constant-index forms.
7979 ConstantSDNode *constIndex;
7980 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7981 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) {
7982 // We need to create a new undef vector to use for the VDUPLANE if the
7983 // size of the vector from which we get the value is different than the
7984 // size of the vector that we need to create. We will insert the element
7985 // such that the register coalescer will remove unnecessary copies.
7986 if (VT != Value->getOperand(0).getValueType()) {
7987 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
7988 VT.getVectorNumElements();
7989 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
7990 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
7991 Value, DAG.getConstant(index, dl, MVT::i32)),
7992 DAG.getConstant(index, dl, MVT::i32));
7993 } else
7994 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
7995 Value->getOperand(0), Value->getOperand(1));
7996 } else
7997 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
7998
7999 if (!usesOnlyOneValue) {
8000 // The dominant value was splatted as 'N', but we now have to insert
8001 // all differing elements.
8002 for (unsigned I = 0; I < NumElts; ++I) {
8003 if (Op.getOperand(I) == Value)
8004 continue;
8005 SmallVector<SDValue, 3> Ops;
8006 Ops.push_back(N);
8007 Ops.push_back(Op.getOperand(I));
8008 Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
8009 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
8010 }
8011 }
8012 return N;
8013 }
8014 if (VT.getVectorElementType().isFloatingPoint()) {
8015 SmallVector<SDValue, 8> Ops;
8016 MVT FVT = VT.getVectorElementType().getSimpleVT();
8017 assert(FVT == MVT::f32 || FVT == MVT::f16);
8018 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8019 for (unsigned i = 0; i < NumElts; ++i)
8020 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, IVT,
8021 Op.getOperand(i)));
8022 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), IVT, NumElts);
8023 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
8024 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8025 if (Val.getNode())
8026 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
8027 }
8028 if (usesOnlyOneValue) {
8029 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
8030 if (isConstant && Val.getNode())
8031 return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
8032 }
8033 }
8034
8035 // If all elements are constants and the case above didn't get hit, fall back
8036 // to the default expansion, which will generate a load from the constant
8037 // pool.
8038 if (isConstant)
8039 return SDValue();
8040
8041 // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
8042 // vmovn). Empirical tests suggest this is rarely worth it for vectors of
8043 // length <= 2.
8044 if (NumElts >= 4)
8045 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
8046 return shuffle;
8047
8048 // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
8049 // VCVT's
8050 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(Op, DAG, Subtarget))
8051 return VCVT;
8052 if (SDValue VCVT = LowerBuildVectorOfFPExt(Op, DAG, Subtarget))
8053 return VCVT;
8054
8055 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8056 // If we haven't found an efficient lowering, try splitting a 128-bit vector
8057 // into two 64-bit vectors; we might discover a better way to lower it.
8058 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
8059 EVT ExtVT = VT.getVectorElementType();
8060 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElts / 2);
8061 SDValue Lower = DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[0], NumElts / 2));
8062 if (Lower.getOpcode() == ISD::BUILD_VECTOR)
8063 Lower = LowerBUILD_VECTOR(Lower, DAG, ST);
8064 SDValue Upper =
8065 DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[NumElts / 2], NumElts / 2));
8066 if (Upper.getOpcode() == ISD::BUILD_VECTOR)
8067 Upper = LowerBUILD_VECTOR(Upper, DAG, ST);
8068 if (Lower && Upper)
8069 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper);
8070 }
8071
8072 // Vectors with 32- or 64-bit elements can be built by directly assigning
8073 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
8074 // will be legalized.
8075 if (EltSize >= 32) {
8076 // Do the expansion with floating-point types, since that is what the VFP
8077 // registers are defined to use, and since i64 is not legal.
8078 EVT EltVT = EVT::getFloatingPointVT(EltSize);
8079 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
8080 SmallVector<SDValue, 8> Ops;
8081 for (unsigned i = 0; i < NumElts; ++i)
8082 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i)));
8083 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
8084 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
8085 }
8086
8087 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
8088 // know the default expansion would otherwise fall back on something even
8089 // worse. For a vector with one or two non-undef values, that's
8090 // scalar_to_vector for the elements followed by a shuffle (provided the
8091 // shuffle is valid for the target) and materialization element by element
8092 // on the stack followed by a load for everything else.
8093 if (!isConstant && !usesOnlyOneValue) {
8094 SDValue Vec = DAG.getUNDEF(VT);
8095 for (unsigned i = 0 ; i < NumElts; ++i) {
8096 SDValue V = Op.getOperand(i);
8097 if (V.isUndef())
8098 continue;
8099 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
8100 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
8101 }
8102 return Vec;
8103 }
8104
8105 return SDValue();
8106 }
8107
8108 // Gather data to see if the operation can be modelled as a
8109 // shuffle in combination with VEXTs.
ReconstructShuffle(SDValue Op,SelectionDAG & DAG) const8110 SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
8111 SelectionDAG &DAG) const {
8112 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
8113 SDLoc dl(Op);
8114 EVT VT = Op.getValueType();
8115 unsigned NumElts = VT.getVectorNumElements();
8116
8117 struct ShuffleSourceInfo {
8118 SDValue Vec;
8119 unsigned MinElt = std::numeric_limits<unsigned>::max();
8120 unsigned MaxElt = 0;
8121
8122 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
8123 // be compatible with the shuffle we intend to construct. As a result
8124 // ShuffleVec will be some sliding window into the original Vec.
8125 SDValue ShuffleVec;
8126
8127 // Code should guarantee that element i in Vec starts at element "WindowBase
8128 // + i * WindowScale in ShuffleVec".
8129 int WindowBase = 0;
8130 int WindowScale = 1;
8131
8132 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8133
8134 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
8135 };
8136
8137 // First gather all vectors used as an immediate source for this BUILD_VECTOR
8138 // node.
8139 SmallVector<ShuffleSourceInfo, 2> Sources;
8140 for (unsigned i = 0; i < NumElts; ++i) {
8141 SDValue V = Op.getOperand(i);
8142 if (V.isUndef())
8143 continue;
8144 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
8145 // A shuffle can only come from building a vector from various
8146 // elements of other vectors.
8147 return SDValue();
8148 } else if (!isa<ConstantSDNode>(V.getOperand(1))) {
8149 // Furthermore, shuffles require a constant mask, whereas extractelts
8150 // accept variable indices.
8151 return SDValue();
8152 }
8153
8154 // Add this element source to the list if it's not already there.
8155 SDValue SourceVec = V.getOperand(0);
8156 auto Source = llvm::find(Sources, SourceVec);
8157 if (Source == Sources.end())
8158 Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
8159
8160 // Update the minimum and maximum lane number seen.
8161 unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
8162 Source->MinElt = std::min(Source->MinElt, EltNo);
8163 Source->MaxElt = std::max(Source->MaxElt, EltNo);
8164 }
8165
8166 // Currently only do something sane when at most two source vectors
8167 // are involved.
8168 if (Sources.size() > 2)
8169 return SDValue();
8170
8171 // Find out the smallest element size among result and two sources, and use
8172 // it as element size to build the shuffle_vector.
8173 EVT SmallestEltTy = VT.getVectorElementType();
8174 for (auto &Source : Sources) {
8175 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
8176 if (SrcEltTy.bitsLT(SmallestEltTy))
8177 SmallestEltTy = SrcEltTy;
8178 }
8179 unsigned ResMultiplier =
8180 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits();
8181 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
8182 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
8183
8184 // If the source vector is too wide or too narrow, we may nevertheless be able
8185 // to construct a compatible shuffle either by concatenating it with UNDEF or
8186 // extracting a suitable range of elements.
8187 for (auto &Src : Sources) {
8188 EVT SrcVT = Src.ShuffleVec.getValueType();
8189
8190 uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
8191 uint64_t VTSize = VT.getFixedSizeInBits();
8192 if (SrcVTSize == VTSize)
8193 continue;
8194
8195 // This stage of the search produces a source with the same element type as
8196 // the original, but with a total width matching the BUILD_VECTOR output.
8197 EVT EltVT = SrcVT.getVectorElementType();
8198 unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
8199 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
8200
8201 if (SrcVTSize < VTSize) {
8202 if (2 * SrcVTSize != VTSize)
8203 return SDValue();
8204 // We can pad out the smaller vector for free, so if it's part of a
8205 // shuffle...
8206 Src.ShuffleVec =
8207 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
8208 DAG.getUNDEF(Src.ShuffleVec.getValueType()));
8209 continue;
8210 }
8211
8212 if (SrcVTSize != 2 * VTSize)
8213 return SDValue();
8214
8215 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8216 // Span too large for a VEXT to cope
8217 return SDValue();
8218 }
8219
8220 if (Src.MinElt >= NumSrcElts) {
8221 // The extraction can just take the second half
8222 Src.ShuffleVec =
8223 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8224 DAG.getConstant(NumSrcElts, dl, MVT::i32));
8225 Src.WindowBase = -NumSrcElts;
8226 } else if (Src.MaxElt < NumSrcElts) {
8227 // The extraction can just take the first half
8228 Src.ShuffleVec =
8229 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8230 DAG.getConstant(0, dl, MVT::i32));
8231 } else {
8232 // An actual VEXT is needed
8233 SDValue VEXTSrc1 =
8234 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8235 DAG.getConstant(0, dl, MVT::i32));
8236 SDValue VEXTSrc2 =
8237 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8238 DAG.getConstant(NumSrcElts, dl, MVT::i32));
8239
8240 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1,
8241 VEXTSrc2,
8242 DAG.getConstant(Src.MinElt, dl, MVT::i32));
8243 Src.WindowBase = -Src.MinElt;
8244 }
8245 }
8246
8247 // Another possible incompatibility occurs from the vector element types. We
8248 // can fix this by bitcasting the source vectors to the same type we intend
8249 // for the shuffle.
8250 for (auto &Src : Sources) {
8251 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8252 if (SrcEltTy == SmallestEltTy)
8253 continue;
8254 assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
8255 Src.ShuffleVec = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, ShuffleVT, Src.ShuffleVec);
8256 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
8257 Src.WindowBase *= Src.WindowScale;
8258 }
8259
8260 // Final check before we try to actually produce a shuffle.
8261 LLVM_DEBUG(for (auto Src
8262 : Sources)
8263 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8264
8265 // The stars all align, our next step is to produce the mask for the shuffle.
8266 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
8267 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
8268 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
8269 SDValue Entry = Op.getOperand(i);
8270 if (Entry.isUndef())
8271 continue;
8272
8273 auto Src = llvm::find(Sources, Entry.getOperand(0));
8274 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
8275
8276 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
8277 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
8278 // segment.
8279 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8280 int BitsDefined = std::min(OrigEltTy.getScalarSizeInBits(),
8281 VT.getScalarSizeInBits());
8282 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8283
8284 // This source is expected to fill ResMultiplier lanes of the final shuffle,
8285 // starting at the appropriate offset.
8286 int *LaneMask = &Mask[i * ResMultiplier];
8287
8288 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8289 ExtractBase += NumElts * (Src - Sources.begin());
8290 for (int j = 0; j < LanesDefined; ++j)
8291 LaneMask[j] = ExtractBase + j;
8292 }
8293
8294
8295 // We can't handle more than two sources. This should have already
8296 // been checked before this point.
8297 assert(Sources.size() <= 2 && "Too many sources!");
8298
8299 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
8300 for (unsigned i = 0; i < Sources.size(); ++i)
8301 ShuffleOps[i] = Sources[i].ShuffleVec;
8302
8303 SDValue Shuffle = buildLegalVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
8304 ShuffleOps[1], Mask, DAG);
8305 if (!Shuffle)
8306 return SDValue();
8307 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuffle);
8308 }
8309
8310 enum ShuffleOpCodes {
8311 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
8312 OP_VREV,
8313 OP_VDUP0,
8314 OP_VDUP1,
8315 OP_VDUP2,
8316 OP_VDUP3,
8317 OP_VEXT1,
8318 OP_VEXT2,
8319 OP_VEXT3,
8320 OP_VUZPL, // VUZP, left result
8321 OP_VUZPR, // VUZP, right result
8322 OP_VZIPL, // VZIP, left result
8323 OP_VZIPR, // VZIP, right result
8324 OP_VTRNL, // VTRN, left result
8325 OP_VTRNR // VTRN, right result
8326 };
8327
isLegalMVEShuffleOp(unsigned PFEntry)8328 static bool isLegalMVEShuffleOp(unsigned PFEntry) {
8329 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8330 switch (OpNum) {
8331 case OP_COPY:
8332 case OP_VREV:
8333 case OP_VDUP0:
8334 case OP_VDUP1:
8335 case OP_VDUP2:
8336 case OP_VDUP3:
8337 return true;
8338 }
8339 return false;
8340 }
8341
8342 /// isShuffleMaskLegal - Targets can use this to indicate that they only
8343 /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
8344 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
8345 /// are assumed to be legal.
isShuffleMaskLegal(ArrayRef<int> M,EVT VT) const8346 bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
8347 if (VT.getVectorNumElements() == 4 &&
8348 (VT.is128BitVector() || VT.is64BitVector())) {
8349 unsigned PFIndexes[4];
8350 for (unsigned i = 0; i != 4; ++i) {
8351 if (M[i] < 0)
8352 PFIndexes[i] = 8;
8353 else
8354 PFIndexes[i] = M[i];
8355 }
8356
8357 // Compute the index in the perfect shuffle table.
8358 unsigned PFTableIndex =
8359 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8360 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8361 unsigned Cost = (PFEntry >> 30);
8362
8363 if (Cost <= 4 && (Subtarget->hasNEON() || isLegalMVEShuffleOp(PFEntry)))
8364 return true;
8365 }
8366
8367 bool ReverseVEXT, isV_UNDEF;
8368 unsigned Imm, WhichResult;
8369
8370 unsigned EltSize = VT.getScalarSizeInBits();
8371 if (EltSize >= 32 ||
8372 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
8373 ShuffleVectorInst::isIdentityMask(M) ||
8374 isVREVMask(M, VT, 64) ||
8375 isVREVMask(M, VT, 32) ||
8376 isVREVMask(M, VT, 16))
8377 return true;
8378 else if (Subtarget->hasNEON() &&
8379 (isVEXTMask(M, VT, ReverseVEXT, Imm) ||
8380 isVTBLMask(M, VT) ||
8381 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF)))
8382 return true;
8383 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8384 isReverseMask(M, VT))
8385 return true;
8386 else if (Subtarget->hasMVEIntegerOps() &&
8387 (isVMOVNMask(M, VT, true, false) ||
8388 isVMOVNMask(M, VT, false, false) || isVMOVNMask(M, VT, true, true)))
8389 return true;
8390 else if (Subtarget->hasMVEIntegerOps() &&
8391 (isTruncMask(M, VT, false, false) ||
8392 isTruncMask(M, VT, false, true) ||
8393 isTruncMask(M, VT, true, false) || isTruncMask(M, VT, true, true)))
8394 return true;
8395 else
8396 return false;
8397 }
8398
8399 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8400 /// the specified operations to build the shuffle.
GeneratePerfectShuffle(unsigned PFEntry,SDValue LHS,SDValue RHS,SelectionDAG & DAG,const SDLoc & dl)8401 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8402 SDValue RHS, SelectionDAG &DAG,
8403 const SDLoc &dl) {
8404 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8405 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8406 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8407
8408 if (OpNum == OP_COPY) {
8409 if (LHSID == (1*9+2)*9+3) return LHS;
8410 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
8411 return RHS;
8412 }
8413
8414 SDValue OpLHS, OpRHS;
8415 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8416 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8417 EVT VT = OpLHS.getValueType();
8418
8419 switch (OpNum) {
8420 default: llvm_unreachable("Unknown shuffle opcode!");
8421 case OP_VREV:
8422 // VREV divides the vector in half and swaps within the half.
8423 if (VT.getScalarSizeInBits() == 32)
8424 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS);
8425 // vrev <4 x i16> -> VREV32
8426 if (VT.getScalarSizeInBits() == 16)
8427 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS);
8428 // vrev <4 x i8> -> VREV16
8429 assert(VT.getScalarSizeInBits() == 8);
8430 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS);
8431 case OP_VDUP0:
8432 case OP_VDUP1:
8433 case OP_VDUP2:
8434 case OP_VDUP3:
8435 return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
8436 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32));
8437 case OP_VEXT1:
8438 case OP_VEXT2:
8439 case OP_VEXT3:
8440 return DAG.getNode(ARMISD::VEXT, dl, VT,
8441 OpLHS, OpRHS,
8442 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32));
8443 case OP_VUZPL:
8444 case OP_VUZPR:
8445 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
8446 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL);
8447 case OP_VZIPL:
8448 case OP_VZIPR:
8449 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
8450 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL);
8451 case OP_VTRNL:
8452 case OP_VTRNR:
8453 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
8454 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL);
8455 }
8456 }
8457
LowerVECTOR_SHUFFLEv8i8(SDValue Op,ArrayRef<int> ShuffleMask,SelectionDAG & DAG)8458 static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
8459 ArrayRef<int> ShuffleMask,
8460 SelectionDAG &DAG) {
8461 // Check to see if we can use the VTBL instruction.
8462 SDValue V1 = Op.getOperand(0);
8463 SDValue V2 = Op.getOperand(1);
8464 SDLoc DL(Op);
8465
8466 SmallVector<SDValue, 8> VTBLMask;
8467 for (int I : ShuffleMask)
8468 VTBLMask.push_back(DAG.getConstant(I, DL, MVT::i32));
8469
8470 if (V2.getNode()->isUndef())
8471 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
8472 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8473
8474 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2,
8475 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8476 }
8477
LowerReverse_VECTOR_SHUFFLE(SDValue Op,SelectionDAG & DAG)8478 static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
8479 SDLoc DL(Op);
8480 EVT VT = Op.getValueType();
8481
8482 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8483 "Expect an v8i16/v16i8 type");
8484 SDValue OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, Op.getOperand(0));
8485 // For a v16i8 type: After the VREV, we have got <7, ..., 0, 15, ..., 8>. Now,
8486 // extract the first 8 bytes into the top double word and the last 8 bytes
8487 // into the bottom double word, through a new vector shuffle that will be
8488 // turned into a VEXT on Neon, or a couple of VMOVDs on MVE.
8489 std::vector<int> NewMask;
8490 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8491 NewMask.push_back(VT.getVectorNumElements() / 2 + i);
8492 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8493 NewMask.push_back(i);
8494 return DAG.getVectorShuffle(VT, DL, OpLHS, OpLHS, NewMask);
8495 }
8496
getVectorTyFromPredicateVector(EVT VT)8497 static EVT getVectorTyFromPredicateVector(EVT VT) {
8498 switch (VT.getSimpleVT().SimpleTy) {
8499 case MVT::v2i1:
8500 return MVT::v2f64;
8501 case MVT::v4i1:
8502 return MVT::v4i32;
8503 case MVT::v8i1:
8504 return MVT::v8i16;
8505 case MVT::v16i1:
8506 return MVT::v16i8;
8507 default:
8508 llvm_unreachable("Unexpected vector predicate type");
8509 }
8510 }
8511
PromoteMVEPredVector(SDLoc dl,SDValue Pred,EVT VT,SelectionDAG & DAG)8512 static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
8513 SelectionDAG &DAG) {
8514 // Converting from boolean predicates to integers involves creating a vector
8515 // of all ones or all zeroes and selecting the lanes based upon the real
8516 // predicate.
8517 SDValue AllOnes =
8518 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff), dl, MVT::i32);
8519 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllOnes);
8520
8521 SDValue AllZeroes =
8522 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0x0), dl, MVT::i32);
8523 AllZeroes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllZeroes);
8524
8525 // Get full vector type from predicate type
8526 EVT NewVT = getVectorTyFromPredicateVector(VT);
8527
8528 SDValue RecastV1;
8529 // If the real predicate is an v8i1 or v4i1 (not v16i1) then we need to recast
8530 // this to a v16i1. This cannot be done with an ordinary bitcast because the
8531 // sizes are not the same. We have to use a MVE specific PREDICATE_CAST node,
8532 // since we know in hardware the sizes are really the same.
8533 if (VT != MVT::v16i1)
8534 RecastV1 = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Pred);
8535 else
8536 RecastV1 = Pred;
8537
8538 // Select either all ones or zeroes depending upon the real predicate bits.
8539 SDValue PredAsVector =
8540 DAG.getNode(ISD::VSELECT, dl, MVT::v16i8, RecastV1, AllOnes, AllZeroes);
8541
8542 // Recast our new predicate-as-integer v16i8 vector into something
8543 // appropriate for the shuffle, i.e. v4i32 for a real v4i1 predicate.
8544 return DAG.getNode(ISD::BITCAST, dl, NewVT, PredAsVector);
8545 }
8546
LowerVECTOR_SHUFFLE_i1(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)8547 static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
8548 const ARMSubtarget *ST) {
8549 EVT VT = Op.getValueType();
8550 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
8551 ArrayRef<int> ShuffleMask = SVN->getMask();
8552
8553 assert(ST->hasMVEIntegerOps() &&
8554 "No support for vector shuffle of boolean predicates");
8555
8556 SDValue V1 = Op.getOperand(0);
8557 SDValue V2 = Op.getOperand(1);
8558 SDLoc dl(Op);
8559 if (isReverseMask(ShuffleMask, VT)) {
8560 SDValue cast = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, V1);
8561 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, cast);
8562 SDValue srl = DAG.getNode(ISD::SRL, dl, MVT::i32, rbit,
8563 DAG.getConstant(16, dl, MVT::i32));
8564 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, srl);
8565 }
8566
8567 // Until we can come up with optimised cases for every single vector
8568 // shuffle in existence we have chosen the least painful strategy. This is
8569 // to essentially promote the boolean predicate to a 8-bit integer, where
8570 // each predicate represents a byte. Then we fall back on a normal integer
8571 // vector shuffle and convert the result back into a predicate vector. In
8572 // many cases the generated code might be even better than scalar code
8573 // operating on bits. Just imagine trying to shuffle 8 arbitrary 2-bit
8574 // fields in a register into 8 other arbitrary 2-bit fields!
8575 SDValue PredAsVector1 = PromoteMVEPredVector(dl, V1, VT, DAG);
8576 EVT NewVT = PredAsVector1.getValueType();
8577 SDValue PredAsVector2 = V2.isUndef() ? DAG.getUNDEF(NewVT)
8578 : PromoteMVEPredVector(dl, V2, VT, DAG);
8579 assert(PredAsVector2.getValueType() == NewVT &&
8580 "Expected identical vector type in expanded i1 shuffle!");
8581
8582 // Do the shuffle!
8583 SDValue Shuffled = DAG.getVectorShuffle(NewVT, dl, PredAsVector1,
8584 PredAsVector2, ShuffleMask);
8585
8586 // Now return the result of comparing the shuffled vector with zero,
8587 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1
8588 // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
8589 if (VT == MVT::v2i1) {
8590 SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Shuffled);
8591 SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC,
8592 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8593 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
8594 }
8595 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled,
8596 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8597 }
8598
LowerVECTOR_SHUFFLEUsingMovs(SDValue Op,ArrayRef<int> ShuffleMask,SelectionDAG & DAG)8599 static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op,
8600 ArrayRef<int> ShuffleMask,
8601 SelectionDAG &DAG) {
8602 // Attempt to lower the vector shuffle using as many whole register movs as
8603 // possible. This is useful for types smaller than 32bits, which would
8604 // often otherwise become a series for grp movs.
8605 SDLoc dl(Op);
8606 EVT VT = Op.getValueType();
8607 if (VT.getScalarSizeInBits() >= 32)
8608 return SDValue();
8609
8610 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8611 "Unexpected vector type");
8612 int NumElts = VT.getVectorNumElements();
8613 int QuarterSize = NumElts / 4;
8614 // The four final parts of the vector, as i32's
8615 SDValue Parts[4];
8616
8617 // Look for full lane vmovs like <0,1,2,3> or <u,5,6,7> etc, (but not
8618 // <u,u,u,u>), returning the vmov lane index
8619 auto getMovIdx = [](ArrayRef<int> ShuffleMask, int Start, int Length) {
8620 // Detect which mov lane this would be from the first non-undef element.
8621 int MovIdx = -1;
8622 for (int i = 0; i < Length; i++) {
8623 if (ShuffleMask[Start + i] >= 0) {
8624 if (ShuffleMask[Start + i] % Length != i)
8625 return -1;
8626 MovIdx = ShuffleMask[Start + i] / Length;
8627 break;
8628 }
8629 }
8630 // If all items are undef, leave this for other combines
8631 if (MovIdx == -1)
8632 return -1;
8633 // Check the remaining values are the correct part of the same mov
8634 for (int i = 1; i < Length; i++) {
8635 if (ShuffleMask[Start + i] >= 0 &&
8636 (ShuffleMask[Start + i] / Length != MovIdx ||
8637 ShuffleMask[Start + i] % Length != i))
8638 return -1;
8639 }
8640 return MovIdx;
8641 };
8642
8643 for (int Part = 0; Part < 4; ++Part) {
8644 // Does this part look like a mov
8645 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8646 if (Elt != -1) {
8647 SDValue Input = Op->getOperand(0);
8648 if (Elt >= 4) {
8649 Input = Op->getOperand(1);
8650 Elt -= 4;
8651 }
8652 SDValue BitCast = DAG.getBitcast(MVT::v4f32, Input);
8653 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, BitCast,
8654 DAG.getConstant(Elt, dl, MVT::i32));
8655 }
8656 }
8657
8658 // Nothing interesting found, just return
8659 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8660 return SDValue();
8661
8662 // The other parts need to be built with the old shuffle vector, cast to a
8663 // v4i32 and extract_vector_elts
8664 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8665 SmallVector<int, 16> NewShuffleMask;
8666 for (int Part = 0; Part < 4; ++Part)
8667 for (int i = 0; i < QuarterSize; i++)
8668 NewShuffleMask.push_back(
8669 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8670 SDValue NewShuffle = DAG.getVectorShuffle(
8671 VT, dl, Op->getOperand(0), Op->getOperand(1), NewShuffleMask);
8672 SDValue BitCast = DAG.getBitcast(MVT::v4f32, NewShuffle);
8673
8674 for (int Part = 0; Part < 4; ++Part)
8675 if (!Parts[Part])
8676 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32,
8677 BitCast, DAG.getConstant(Part, dl, MVT::i32));
8678 }
8679 // Build a vector out of the various parts and bitcast it back to the original
8680 // type.
8681 SDValue NewVec = DAG.getNode(ARMISD::BUILD_VECTOR, dl, MVT::v4f32, Parts);
8682 return DAG.getBitcast(VT, NewVec);
8683 }
8684
LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op,ArrayRef<int> ShuffleMask,SelectionDAG & DAG)8685 static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op,
8686 ArrayRef<int> ShuffleMask,
8687 SelectionDAG &DAG) {
8688 SDValue V1 = Op.getOperand(0);
8689 SDValue V2 = Op.getOperand(1);
8690 EVT VT = Op.getValueType();
8691 unsigned NumElts = VT.getVectorNumElements();
8692
8693 // An One-Off Identity mask is one that is mostly an identity mask from as
8694 // single source but contains a single element out-of-place, either from a
8695 // different vector or from another position in the same vector. As opposed to
8696 // lowering this via a ARMISD::BUILD_VECTOR we can generate an extract/insert
8697 // pair directly.
8698 auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset,
8699 int &OffElement) {
8700 OffElement = -1;
8701 int NonUndef = 0;
8702 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8703 if (Mask[i] == -1)
8704 continue;
8705 NonUndef++;
8706 if (Mask[i] != i + BaseOffset) {
8707 if (OffElement == -1)
8708 OffElement = i;
8709 else
8710 return false;
8711 }
8712 }
8713 return NonUndef > 2 && OffElement != -1;
8714 };
8715 int OffElement;
8716 SDValue VInput;
8717 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8718 VInput = V1;
8719 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8720 VInput = V2;
8721 else
8722 return SDValue();
8723
8724 SDLoc dl(Op);
8725 EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16
8726 ? MVT::i32
8727 : VT.getScalarType();
8728 SDValue Elt = DAG.getNode(
8729 ISD::EXTRACT_VECTOR_ELT, dl, SVT,
8730 ShuffleMask[OffElement] < (int)NumElts ? V1 : V2,
8731 DAG.getVectorIdxConstant(ShuffleMask[OffElement] % NumElts, dl));
8732 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, VInput, Elt,
8733 DAG.getVectorIdxConstant(OffElement % NumElts, dl));
8734 }
8735
LowerVECTOR_SHUFFLE(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)8736 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
8737 const ARMSubtarget *ST) {
8738 SDValue V1 = Op.getOperand(0);
8739 SDValue V2 = Op.getOperand(1);
8740 SDLoc dl(Op);
8741 EVT VT = Op.getValueType();
8742 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
8743 unsigned EltSize = VT.getScalarSizeInBits();
8744
8745 if (ST->hasMVEIntegerOps() && EltSize == 1)
8746 return LowerVECTOR_SHUFFLE_i1(Op, DAG, ST);
8747
8748 // Convert shuffles that are directly supported on NEON to target-specific
8749 // DAG nodes, instead of keeping them as shuffles and matching them again
8750 // during code selection. This is more efficient and avoids the possibility
8751 // of inconsistencies between legalization and selection.
8752 // FIXME: floating-point vectors should be canonicalized to integer vectors
8753 // of the same time so that they get CSEd properly.
8754 ArrayRef<int> ShuffleMask = SVN->getMask();
8755
8756 if (EltSize <= 32) {
8757 if (SVN->isSplat()) {
8758 int Lane = SVN->getSplatIndex();
8759 // If this is undef splat, generate it via "just" vdup, if possible.
8760 if (Lane == -1) Lane = 0;
8761
8762 // Test if V1 is a SCALAR_TO_VECTOR.
8763 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
8764 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8765 }
8766 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
8767 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
8768 // reaches it).
8769 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
8770 !isa<ConstantSDNode>(V1.getOperand(0))) {
8771 bool IsScalarToVector = true;
8772 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
8773 if (!V1.getOperand(i).isUndef()) {
8774 IsScalarToVector = false;
8775 break;
8776 }
8777 if (IsScalarToVector)
8778 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8779 }
8780 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
8781 DAG.getConstant(Lane, dl, MVT::i32));
8782 }
8783
8784 bool ReverseVEXT = false;
8785 unsigned Imm = 0;
8786 if (ST->hasNEON() && isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8787 if (ReverseVEXT)
8788 std::swap(V1, V2);
8789 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
8790 DAG.getConstant(Imm, dl, MVT::i32));
8791 }
8792
8793 if (isVREVMask(ShuffleMask, VT, 64))
8794 return DAG.getNode(ARMISD::VREV64, dl, VT, V1);
8795 if (isVREVMask(ShuffleMask, VT, 32))
8796 return DAG.getNode(ARMISD::VREV32, dl, VT, V1);
8797 if (isVREVMask(ShuffleMask, VT, 16))
8798 return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
8799
8800 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) {
8801 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
8802 DAG.getConstant(Imm, dl, MVT::i32));
8803 }
8804
8805 // Check for Neon shuffles that modify both input vectors in place.
8806 // If both results are used, i.e., if there are two shuffles with the same
8807 // source operands and with masks corresponding to both results of one of
8808 // these operations, DAG memoization will ensure that a single node is
8809 // used for both shuffles.
8810 unsigned WhichResult = 0;
8811 bool isV_UNDEF = false;
8812 if (ST->hasNEON()) {
8813 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8814 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8815 if (isV_UNDEF)
8816 V2 = V1;
8817 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
8818 .getValue(WhichResult);
8819 }
8820 }
8821 if (ST->hasMVEIntegerOps()) {
8822 if (isVMOVNMask(ShuffleMask, VT, false, false))
8823 return DAG.getNode(ARMISD::VMOVN, dl, VT, V2, V1,
8824 DAG.getConstant(0, dl, MVT::i32));
8825 if (isVMOVNMask(ShuffleMask, VT, true, false))
8826 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V2,
8827 DAG.getConstant(1, dl, MVT::i32));
8828 if (isVMOVNMask(ShuffleMask, VT, true, true))
8829 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V1,
8830 DAG.getConstant(1, dl, MVT::i32));
8831 }
8832
8833 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
8834 // shuffles that produce a result larger than their operands with:
8835 // shuffle(concat(v1, undef), concat(v2, undef))
8836 // ->
8837 // shuffle(concat(v1, v2), undef)
8838 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
8839 //
8840 // This is useful in the general case, but there are special cases where
8841 // native shuffles produce larger results: the two-result ops.
8842 //
8843 // Look through the concat when lowering them:
8844 // shuffle(concat(v1, v2), undef)
8845 // ->
8846 // concat(VZIP(v1, v2):0, :1)
8847 //
8848 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) {
8849 SDValue SubV1 = V1->getOperand(0);
8850 SDValue SubV2 = V1->getOperand(1);
8851 EVT SubVT = SubV1.getValueType();
8852
8853 // We expect these to have been canonicalized to -1.
8854 assert(llvm::all_of(ShuffleMask, [&](int i) {
8855 return i < (int)VT.getVectorNumElements();
8856 }) && "Unexpected shuffle index into UNDEF operand!");
8857
8858 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8859 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8860 if (isV_UNDEF)
8861 SubV2 = SubV1;
8862 assert((WhichResult == 0) &&
8863 "In-place shuffle of concat can only have one result!");
8864 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
8865 SubV1, SubV2);
8866 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
8867 Res.getValue(1));
8868 }
8869 }
8870 }
8871
8872 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8873 if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG))
8874 return V;
8875
8876 for (bool Top : {false, true}) {
8877 for (bool SingleSource : {false, true}) {
8878 if (isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8879 MVT FromSVT = MVT::getIntegerVT(EltSize * 2);
8880 MVT FromVT = MVT::getVectorVT(FromSVT, ShuffleMask.size() / 2);
8881 SDValue Lo = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, FromVT, V1);
8882 SDValue Hi = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, FromVT,
8883 SingleSource ? V1 : V2);
8884 if (Top) {
8885 SDValue Amt = DAG.getConstant(EltSize, dl, FromVT);
8886 Lo = DAG.getNode(ISD::SRL, dl, FromVT, Lo, Amt);
8887 Hi = DAG.getNode(ISD::SRL, dl, FromVT, Hi, Amt);
8888 }
8889 return DAG.getNode(ARMISD::MVETRUNC, dl, VT, Lo, Hi);
8890 }
8891 }
8892 }
8893 }
8894
8895 // If the shuffle is not directly supported and it has 4 elements, use
8896 // the PerfectShuffle-generated table to synthesize it from other shuffles.
8897 unsigned NumElts = VT.getVectorNumElements();
8898 if (NumElts == 4) {
8899 unsigned PFIndexes[4];
8900 for (unsigned i = 0; i != 4; ++i) {
8901 if (ShuffleMask[i] < 0)
8902 PFIndexes[i] = 8;
8903 else
8904 PFIndexes[i] = ShuffleMask[i];
8905 }
8906
8907 // Compute the index in the perfect shuffle table.
8908 unsigned PFTableIndex =
8909 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8910 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8911 unsigned Cost = (PFEntry >> 30);
8912
8913 if (Cost <= 4) {
8914 if (ST->hasNEON())
8915 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
8916 else if (isLegalMVEShuffleOp(PFEntry)) {
8917 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8918 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8919 unsigned PFEntryLHS = PerfectShuffleTable[LHSID];
8920 unsigned PFEntryRHS = PerfectShuffleTable[RHSID];
8921 if (isLegalMVEShuffleOp(PFEntryLHS) && isLegalMVEShuffleOp(PFEntryRHS))
8922 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
8923 }
8924 }
8925 }
8926
8927 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
8928 if (EltSize >= 32) {
8929 // Do the expansion with floating-point types, since that is what the VFP
8930 // registers are defined to use, and since i64 is not legal.
8931 EVT EltVT = EVT::getFloatingPointVT(EltSize);
8932 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
8933 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1);
8934 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2);
8935 SmallVector<SDValue, 8> Ops;
8936 for (unsigned i = 0; i < NumElts; ++i) {
8937 if (ShuffleMask[i] < 0)
8938 Ops.push_back(DAG.getUNDEF(EltVT));
8939 else
8940 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
8941 ShuffleMask[i] < (int)NumElts ? V1 : V2,
8942 DAG.getConstant(ShuffleMask[i] & (NumElts-1),
8943 dl, MVT::i32)));
8944 }
8945 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
8946 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
8947 }
8948
8949 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8950 isReverseMask(ShuffleMask, VT))
8951 return LowerReverse_VECTOR_SHUFFLE(Op, DAG);
8952
8953 if (ST->hasNEON() && VT == MVT::v8i8)
8954 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
8955 return NewOp;
8956
8957 if (ST->hasMVEIntegerOps())
8958 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
8959 return NewOp;
8960
8961 return SDValue();
8962 }
8963
LowerINSERT_VECTOR_ELT_i1(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)8964 static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8965 const ARMSubtarget *ST) {
8966 EVT VecVT = Op.getOperand(0).getValueType();
8967 SDLoc dl(Op);
8968
8969 assert(ST->hasMVEIntegerOps() &&
8970 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8971
8972 SDValue Conv =
8973 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
8974 unsigned Lane = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
8975 unsigned LaneWidth =
8976 getVectorTyFromPredicateVector(VecVT).getScalarSizeInBits() / 8;
8977 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
8978 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32,
8979 Op.getOperand(1), DAG.getValueType(MVT::i1));
8980 SDValue BFI = DAG.getNode(ARMISD::BFI, dl, MVT::i32, Conv, Ext,
8981 DAG.getConstant(~Mask, dl, MVT::i32));
8982 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), BFI);
8983 }
8984
LowerINSERT_VECTOR_ELT(SDValue Op,SelectionDAG & DAG) const8985 SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8986 SelectionDAG &DAG) const {
8987 // INSERT_VECTOR_ELT is legal only for immediate indexes.
8988 SDValue Lane = Op.getOperand(2);
8989 if (!isa<ConstantSDNode>(Lane))
8990 return SDValue();
8991
8992 SDValue Elt = Op.getOperand(1);
8993 EVT EltVT = Elt.getValueType();
8994
8995 if (Subtarget->hasMVEIntegerOps() &&
8996 Op.getValueType().getScalarSizeInBits() == 1)
8997 return LowerINSERT_VECTOR_ELT_i1(Op, DAG, Subtarget);
8998
8999 if (getTypeAction(*DAG.getContext(), EltVT) ==
9000 TargetLowering::TypePromoteFloat) {
9001 // INSERT_VECTOR_ELT doesn't want f16 operands promoting to f32,
9002 // but the type system will try to do that if we don't intervene.
9003 // Reinterpret any such vector-element insertion as one with the
9004 // corresponding integer types.
9005
9006 SDLoc dl(Op);
9007
9008 EVT IEltVT = MVT::getIntegerVT(EltVT.getScalarSizeInBits());
9009 assert(getTypeAction(*DAG.getContext(), IEltVT) !=
9010 TargetLowering::TypePromoteFloat);
9011
9012 SDValue VecIn = Op.getOperand(0);
9013 EVT VecVT = VecIn.getValueType();
9014 EVT IVecVT = EVT::getVectorVT(*DAG.getContext(), IEltVT,
9015 VecVT.getVectorNumElements());
9016
9017 SDValue IElt = DAG.getNode(ISD::BITCAST, dl, IEltVT, Elt);
9018 SDValue IVecIn = DAG.getNode(ISD::BITCAST, dl, IVecVT, VecIn);
9019 SDValue IVecOut = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVecVT,
9020 IVecIn, IElt, Lane);
9021 return DAG.getNode(ISD::BITCAST, dl, VecVT, IVecOut);
9022 }
9023
9024 return Op;
9025 }
9026
LowerEXTRACT_VECTOR_ELT_i1(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)9027 static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
9028 const ARMSubtarget *ST) {
9029 EVT VecVT = Op.getOperand(0).getValueType();
9030 SDLoc dl(Op);
9031
9032 assert(ST->hasMVEIntegerOps() &&
9033 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9034
9035 SDValue Conv =
9036 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
9037 unsigned Lane = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
9038 unsigned LaneWidth =
9039 getVectorTyFromPredicateVector(VecVT).getScalarSizeInBits() / 8;
9040 SDValue Shift = DAG.getNode(ISD::SRL, dl, MVT::i32, Conv,
9041 DAG.getConstant(Lane * LaneWidth, dl, MVT::i32));
9042 return Shift;
9043 }
9044
LowerEXTRACT_VECTOR_ELT(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)9045 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
9046 const ARMSubtarget *ST) {
9047 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
9048 SDValue Lane = Op.getOperand(1);
9049 if (!isa<ConstantSDNode>(Lane))
9050 return SDValue();
9051
9052 SDValue Vec = Op.getOperand(0);
9053 EVT VT = Vec.getValueType();
9054
9055 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
9056 return LowerEXTRACT_VECTOR_ELT_i1(Op, DAG, ST);
9057
9058 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) {
9059 SDLoc dl(Op);
9060 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
9061 }
9062
9063 return Op;
9064 }
9065
LowerCONCAT_VECTORS_i1(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)9066 static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
9067 const ARMSubtarget *ST) {
9068 SDLoc dl(Op);
9069 assert(Op.getValueType().getScalarSizeInBits() == 1 &&
9070 "Unexpected custom CONCAT_VECTORS lowering");
9071 assert(isPowerOf2_32(Op.getNumOperands()) &&
9072 "Unexpected custom CONCAT_VECTORS lowering");
9073 assert(ST->hasMVEIntegerOps() &&
9074 "CONCAT_VECTORS lowering only supported for MVE");
9075
9076 auto ConcatPair = [&](SDValue V1, SDValue V2) {
9077 EVT Op1VT = V1.getValueType();
9078 EVT Op2VT = V2.getValueType();
9079 assert(Op1VT == Op2VT && "Operand types don't match!");
9080 EVT VT = Op1VT.getDoubleNumVectorElementsVT(*DAG.getContext());
9081
9082 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
9083 SDValue NewV2 = PromoteMVEPredVector(dl, V2, Op2VT, DAG);
9084
9085 // We now have Op1 + Op2 promoted to vectors of integers, where v8i1 gets
9086 // promoted to v8i16, etc.
9087 MVT ElType =
9088 getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
9089 unsigned NumElts = 2 * Op1VT.getVectorNumElements();
9090
9091 // Extract the vector elements from Op1 and Op2 one by one and truncate them
9092 // to be the right size for the destination. For example, if Op1 is v4i1
9093 // then the promoted vector is v4i32. The result of concatenation gives a
9094 // v8i1, which when promoted is v8i16. That means each i32 element from Op1
9095 // needs truncating to i16 and inserting in the result.
9096 EVT ConcatVT = MVT::getVectorVT(ElType, NumElts);
9097 SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
9098 auto ExtractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
9099 EVT NewVT = NewV.getValueType();
9100 EVT ConcatVT = ConVec.getValueType();
9101 for (unsigned i = 0, e = NewVT.getVectorNumElements(); i < e; i++, j++) {
9102 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV,
9103 DAG.getIntPtrConstant(i, dl));
9104 ConVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ConcatVT, ConVec, Elt,
9105 DAG.getConstant(j, dl, MVT::i32));
9106 }
9107 return ConVec;
9108 };
9109 unsigned j = 0;
9110 ConVec = ExtractInto(NewV1, ConVec, j);
9111 ConVec = ExtractInto(NewV2, ConVec, j);
9112
9113 // Now return the result of comparing the subvector with zero, which will
9114 // generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1 we
9115 // convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
9116 if (VT == MVT::v2i1) {
9117 SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, ConVec);
9118 SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC,
9119 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9120 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
9121 }
9122 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
9123 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9124 };
9125
9126 // Concat each pair of subvectors and pack into the lower half of the array.
9127 SmallVector<SDValue> ConcatOps(Op->op_begin(), Op->op_end());
9128 while (ConcatOps.size() > 1) {
9129 for (unsigned I = 0, E = ConcatOps.size(); I != E; I += 2) {
9130 SDValue V1 = ConcatOps[I];
9131 SDValue V2 = ConcatOps[I + 1];
9132 ConcatOps[I / 2] = ConcatPair(V1, V2);
9133 }
9134 ConcatOps.resize(ConcatOps.size() / 2);
9135 }
9136 return ConcatOps[0];
9137 }
9138
LowerCONCAT_VECTORS(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)9139 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9140 const ARMSubtarget *ST) {
9141 EVT VT = Op->getValueType(0);
9142 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
9143 return LowerCONCAT_VECTORS_i1(Op, DAG, ST);
9144
9145 // The only time a CONCAT_VECTORS operation can have legal types is when
9146 // two 64-bit vectors are concatenated to a 128-bit vector.
9147 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
9148 "unexpected CONCAT_VECTORS");
9149 SDLoc dl(Op);
9150 SDValue Val = DAG.getUNDEF(MVT::v2f64);
9151 SDValue Op0 = Op.getOperand(0);
9152 SDValue Op1 = Op.getOperand(1);
9153 if (!Op0.isUndef())
9154 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
9155 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0),
9156 DAG.getIntPtrConstant(0, dl));
9157 if (!Op1.isUndef())
9158 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
9159 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1),
9160 DAG.getIntPtrConstant(1, dl));
9161 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val);
9162 }
9163
LowerEXTRACT_SUBVECTOR(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)9164 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
9165 const ARMSubtarget *ST) {
9166 SDValue V1 = Op.getOperand(0);
9167 SDValue V2 = Op.getOperand(1);
9168 SDLoc dl(Op);
9169 EVT VT = Op.getValueType();
9170 EVT Op1VT = V1.getValueType();
9171 unsigned NumElts = VT.getVectorNumElements();
9172 unsigned Index = cast<ConstantSDNode>(V2)->getZExtValue();
9173
9174 assert(VT.getScalarSizeInBits() == 1 &&
9175 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9176 assert(ST->hasMVEIntegerOps() &&
9177 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9178
9179 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
9180
9181 // We now have Op1 promoted to a vector of integers, where v8i1 gets
9182 // promoted to v8i16, etc.
9183
9184 MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
9185
9186 if (NumElts == 2) {
9187 EVT SubVT = MVT::v4i32;
9188 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
9189 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
9190 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
9191 DAG.getIntPtrConstant(i, dl));
9192 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
9193 DAG.getConstant(j, dl, MVT::i32));
9194 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
9195 DAG.getConstant(j + 1, dl, MVT::i32));
9196 }
9197 SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, SubVec,
9198 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9199 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
9200 }
9201
9202 EVT SubVT = MVT::getVectorVT(ElType, NumElts);
9203 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
9204 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
9205 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
9206 DAG.getIntPtrConstant(i, dl));
9207 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
9208 DAG.getConstant(j, dl, MVT::i32));
9209 }
9210
9211 // Now return the result of comparing the subvector with zero,
9212 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
9213 return DAG.getNode(ARMISD::VCMPZ, dl, VT, SubVec,
9214 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9215 }
9216
9217 // Turn a truncate into a predicate (an i1 vector) into icmp(and(x, 1), 0).
LowerTruncatei1(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)9218 static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG,
9219 const ARMSubtarget *ST) {
9220 assert(ST->hasMVEIntegerOps() && "Expected MVE!");
9221 EVT VT = N->getValueType(0);
9222 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9223 "Expected a vector i1 type!");
9224 SDValue Op = N->getOperand(0);
9225 EVT FromVT = Op.getValueType();
9226 SDLoc DL(N);
9227
9228 SDValue And =
9229 DAG.getNode(ISD::AND, DL, FromVT, Op, DAG.getConstant(1, DL, FromVT));
9230 return DAG.getNode(ISD::SETCC, DL, VT, And, DAG.getConstant(0, DL, FromVT),
9231 DAG.getCondCode(ISD::SETNE));
9232 }
9233
LowerTruncate(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)9234 static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG,
9235 const ARMSubtarget *Subtarget) {
9236 if (!Subtarget->hasMVEIntegerOps())
9237 return SDValue();
9238
9239 EVT ToVT = N->getValueType(0);
9240 if (ToVT.getScalarType() == MVT::i1)
9241 return LowerTruncatei1(N, DAG, Subtarget);
9242
9243 // MVE does not have a single instruction to perform the truncation of a v4i32
9244 // into the lower half of a v8i16, in the same way that a NEON vmovn would.
9245 // Most of the instructions in MVE follow the 'Beats' system, where moving
9246 // values from different lanes is usually something that the instructions
9247 // avoid.
9248 //
9249 // Instead it has top/bottom instructions such as VMOVLT/B and VMOVNT/B,
9250 // which take a the top/bottom half of a larger lane and extend it (or do the
9251 // opposite, truncating into the top/bottom lane from a larger lane). Note
9252 // that because of the way we widen lanes, a v4i16 is really a v4i32 using the
9253 // bottom 16bits from each vector lane. This works really well with T/B
9254 // instructions, but that doesn't extend to v8i32->v8i16 where the lanes need
9255 // to move order.
9256 //
9257 // But truncates and sext/zext are always going to be fairly common from llvm.
9258 // We have several options for how to deal with them:
9259 // - Wherever possible combine them into an instruction that makes them
9260 // "free". This includes loads/stores, which can perform the trunc as part
9261 // of the memory operation. Or certain shuffles that can be turned into
9262 // VMOVN/VMOVL.
9263 // - Lane Interleaving to transform blocks surrounded by ext/trunc. So
9264 // trunc(mul(sext(a), sext(b))) may become
9265 // VMOVNT(VMUL(VMOVLB(a), VMOVLB(b)), VMUL(VMOVLT(a), VMOVLT(b))). (Which in
9266 // this case can use VMULL). This is performed in the
9267 // MVELaneInterleavingPass.
9268 // - Otherwise we have an option. By default we would expand the
9269 // zext/sext/trunc into a series of lane extract/inserts going via GPR
9270 // registers. One for each vector lane in the vector. This can obviously be
9271 // very expensive.
9272 // - The other option is to use the fact that loads/store can extend/truncate
9273 // to turn a trunc into two truncating stack stores and a stack reload. This
9274 // becomes 3 back-to-back memory operations, but at least that is less than
9275 // all the insert/extracts.
9276 //
9277 // In order to do the last, we convert certain trunc's into MVETRUNC, which
9278 // are either optimized where they can be, or eventually lowered into stack
9279 // stores/loads. This prevents us from splitting a v8i16 trunc into two stores
9280 // two early, where other instructions would be better, and stops us from
9281 // having to reconstruct multiple buildvector shuffles into loads/stores.
9282 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9283 return SDValue();
9284 EVT FromVT = N->getOperand(0).getValueType();
9285 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9286 return SDValue();
9287
9288 SDValue Lo, Hi;
9289 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
9290 SDLoc DL(N);
9291 return DAG.getNode(ARMISD::MVETRUNC, DL, ToVT, Lo, Hi);
9292 }
9293
LowerVectorExtend(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)9294 static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG,
9295 const ARMSubtarget *Subtarget) {
9296 if (!Subtarget->hasMVEIntegerOps())
9297 return SDValue();
9298
9299 // See LowerTruncate above for an explanation of MVEEXT/MVETRUNC.
9300
9301 EVT ToVT = N->getValueType(0);
9302 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9303 return SDValue();
9304 SDValue Op = N->getOperand(0);
9305 EVT FromVT = Op.getValueType();
9306 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9307 return SDValue();
9308
9309 SDLoc DL(N);
9310 EVT ExtVT = ToVT.getHalfNumVectorElementsVT(*DAG.getContext());
9311 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8)
9312 ExtVT = MVT::v8i16;
9313
9314 unsigned Opcode =
9315 N->getOpcode() == ISD::SIGN_EXTEND ? ARMISD::MVESEXT : ARMISD::MVEZEXT;
9316 SDValue Ext = DAG.getNode(Opcode, DL, DAG.getVTList(ExtVT, ExtVT), Op);
9317 SDValue Ext1 = Ext.getValue(1);
9318
9319 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8) {
9320 Ext = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext);
9321 Ext1 = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext1);
9322 }
9323
9324 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Ext, Ext1);
9325 }
9326
9327 /// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
9328 /// element has been zero/sign-extended, depending on the isSigned parameter,
9329 /// from an integer type half its size.
isExtendedBUILD_VECTOR(SDNode * N,SelectionDAG & DAG,bool isSigned)9330 static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
9331 bool isSigned) {
9332 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
9333 EVT VT = N->getValueType(0);
9334 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
9335 SDNode *BVN = N->getOperand(0).getNode();
9336 if (BVN->getValueType(0) != MVT::v4i32 ||
9337 BVN->getOpcode() != ISD::BUILD_VECTOR)
9338 return false;
9339 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9340 unsigned HiElt = 1 - LoElt;
9341 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt));
9342 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt));
9343 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2));
9344 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2));
9345 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9346 return false;
9347 if (isSigned) {
9348 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
9349 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
9350 return true;
9351 } else {
9352 if (Hi0->isZero() && Hi1->isZero())
9353 return true;
9354 }
9355 return false;
9356 }
9357
9358 if (N->getOpcode() != ISD::BUILD_VECTOR)
9359 return false;
9360
9361 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
9362 SDNode *Elt = N->getOperand(i).getNode();
9363 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
9364 unsigned EltSize = VT.getScalarSizeInBits();
9365 unsigned HalfSize = EltSize / 2;
9366 if (isSigned) {
9367 if (!isIntN(HalfSize, C->getSExtValue()))
9368 return false;
9369 } else {
9370 if (!isUIntN(HalfSize, C->getZExtValue()))
9371 return false;
9372 }
9373 continue;
9374 }
9375 return false;
9376 }
9377
9378 return true;
9379 }
9380
9381 /// isSignExtended - Check if a node is a vector value that is sign-extended
9382 /// or a constant BUILD_VECTOR with sign-extended elements.
isSignExtended(SDNode * N,SelectionDAG & DAG)9383 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
9384 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
9385 return true;
9386 if (isExtendedBUILD_VECTOR(N, DAG, true))
9387 return true;
9388 return false;
9389 }
9390
9391 /// isZeroExtended - Check if a node is a vector value that is zero-extended (or
9392 /// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
isZeroExtended(SDNode * N,SelectionDAG & DAG)9393 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
9394 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND ||
9395 ISD::isZEXTLoad(N))
9396 return true;
9397 if (isExtendedBUILD_VECTOR(N, DAG, false))
9398 return true;
9399 return false;
9400 }
9401
getExtensionTo64Bits(const EVT & OrigVT)9402 static EVT getExtensionTo64Bits(const EVT &OrigVT) {
9403 if (OrigVT.getSizeInBits() >= 64)
9404 return OrigVT;
9405
9406 assert(OrigVT.isSimple() && "Expecting a simple value type");
9407
9408 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
9409 switch (OrigSimpleTy) {
9410 default: llvm_unreachable("Unexpected Vector Type");
9411 case MVT::v2i8:
9412 case MVT::v2i16:
9413 return MVT::v2i32;
9414 case MVT::v4i8:
9415 return MVT::v4i16;
9416 }
9417 }
9418
9419 /// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
9420 /// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
9421 /// We insert the required extension here to get the vector to fill a D register.
AddRequiredExtensionForVMULL(SDValue N,SelectionDAG & DAG,const EVT & OrigTy,const EVT & ExtTy,unsigned ExtOpcode)9422 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
9423 const EVT &OrigTy,
9424 const EVT &ExtTy,
9425 unsigned ExtOpcode) {
9426 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
9427 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
9428 // 64-bits we need to insert a new extension so that it will be 64-bits.
9429 assert(ExtTy.is128BitVector() && "Unexpected extension size");
9430 if (OrigTy.getSizeInBits() >= 64)
9431 return N;
9432
9433 // Must extend size to at least 64 bits to be used as an operand for VMULL.
9434 EVT NewVT = getExtensionTo64Bits(OrigTy);
9435
9436 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
9437 }
9438
9439 /// SkipLoadExtensionForVMULL - return a load of the original vector size that
9440 /// does not do any sign/zero extension. If the original vector is less
9441 /// than 64 bits, an appropriate extension will be added after the load to
9442 /// reach a total size of 64 bits. We have to add the extension separately
9443 /// because ARM does not have a sign/zero extending load for vectors.
SkipLoadExtensionForVMULL(LoadSDNode * LD,SelectionDAG & DAG)9444 static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
9445 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT());
9446
9447 // The load already has the right type.
9448 if (ExtendedTy == LD->getMemoryVT())
9449 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(),
9450 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9451 LD->getMemOperand()->getFlags());
9452
9453 // We need to create a zextload/sextload. We cannot just create a load
9454 // followed by a zext/zext node because LowerMUL is also run during normal
9455 // operation legalization where we can't create illegal types.
9456 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy,
9457 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9458 LD->getMemoryVT(), LD->getAlign(),
9459 LD->getMemOperand()->getFlags());
9460 }
9461
9462 /// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
9463 /// ANY_EXTEND, extending load, or BUILD_VECTOR with extended elements, return
9464 /// the unextended value. The unextended vector should be 64 bits so that it can
9465 /// be used as an operand to a VMULL instruction. If the original vector size
9466 /// before extension is less than 64 bits we add a an extension to resize
9467 /// the vector to 64 bits.
SkipExtensionForVMULL(SDNode * N,SelectionDAG & DAG)9468 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
9469 if (N->getOpcode() == ISD::SIGN_EXTEND ||
9470 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
9471 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG,
9472 N->getOperand(0)->getValueType(0),
9473 N->getValueType(0),
9474 N->getOpcode());
9475
9476 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
9477 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) &&
9478 "Expected extending load");
9479
9480 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
9481 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), newLoad.getValue(1));
9482 unsigned Opcode = ISD::isSEXTLoad(LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9483 SDValue extLoad =
9484 DAG.getNode(Opcode, SDLoc(newLoad), LD->getValueType(0), newLoad);
9485 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 0), extLoad);
9486
9487 return newLoad;
9488 }
9489
9490 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
9491 // have been legalized as a BITCAST from v4i32.
9492 if (N->getOpcode() == ISD::BITCAST) {
9493 SDNode *BVN = N->getOperand(0).getNode();
9494 assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&
9495 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
9496 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9497 return DAG.getBuildVector(
9498 MVT::v2i32, SDLoc(N),
9499 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)});
9500 }
9501 // Construct a new BUILD_VECTOR with elements truncated to half the size.
9502 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
9503 EVT VT = N->getValueType(0);
9504 unsigned EltSize = VT.getScalarSizeInBits() / 2;
9505 unsigned NumElts = VT.getVectorNumElements();
9506 MVT TruncVT = MVT::getIntegerVT(EltSize);
9507 SmallVector<SDValue, 8> Ops;
9508 SDLoc dl(N);
9509 for (unsigned i = 0; i != NumElts; ++i) {
9510 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
9511 const APInt &CInt = C->getAPIntValue();
9512 // Element types smaller than 32 bits are not legal, so use i32 elements.
9513 // The values are implicitly truncated so sext vs. zext doesn't matter.
9514 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
9515 }
9516 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
9517 }
9518
isAddSubSExt(SDNode * N,SelectionDAG & DAG)9519 static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
9520 unsigned Opcode = N->getOpcode();
9521 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9522 SDNode *N0 = N->getOperand(0).getNode();
9523 SDNode *N1 = N->getOperand(1).getNode();
9524 return N0->hasOneUse() && N1->hasOneUse() &&
9525 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
9526 }
9527 return false;
9528 }
9529
isAddSubZExt(SDNode * N,SelectionDAG & DAG)9530 static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
9531 unsigned Opcode = N->getOpcode();
9532 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9533 SDNode *N0 = N->getOperand(0).getNode();
9534 SDNode *N1 = N->getOperand(1).getNode();
9535 return N0->hasOneUse() && N1->hasOneUse() &&
9536 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
9537 }
9538 return false;
9539 }
9540
LowerMUL(SDValue Op,SelectionDAG & DAG)9541 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
9542 // Multiplications are only custom-lowered for 128-bit vectors so that
9543 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
9544 EVT VT = Op.getValueType();
9545 assert(VT.is128BitVector() && VT.isInteger() &&
9546 "unexpected type for custom-lowering ISD::MUL");
9547 SDNode *N0 = Op.getOperand(0).getNode();
9548 SDNode *N1 = Op.getOperand(1).getNode();
9549 unsigned NewOpc = 0;
9550 bool isMLA = false;
9551 bool isN0SExt = isSignExtended(N0, DAG);
9552 bool isN1SExt = isSignExtended(N1, DAG);
9553 if (isN0SExt && isN1SExt)
9554 NewOpc = ARMISD::VMULLs;
9555 else {
9556 bool isN0ZExt = isZeroExtended(N0, DAG);
9557 bool isN1ZExt = isZeroExtended(N1, DAG);
9558 if (isN0ZExt && isN1ZExt)
9559 NewOpc = ARMISD::VMULLu;
9560 else if (isN1SExt || isN1ZExt) {
9561 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
9562 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
9563 if (isN1SExt && isAddSubSExt(N0, DAG)) {
9564 NewOpc = ARMISD::VMULLs;
9565 isMLA = true;
9566 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
9567 NewOpc = ARMISD::VMULLu;
9568 isMLA = true;
9569 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
9570 std::swap(N0, N1);
9571 NewOpc = ARMISD::VMULLu;
9572 isMLA = true;
9573 }
9574 }
9575
9576 if (!NewOpc) {
9577 if (VT == MVT::v2i64)
9578 // Fall through to expand this. It is not legal.
9579 return SDValue();
9580 else
9581 // Other vector multiplications are legal.
9582 return Op;
9583 }
9584 }
9585
9586 // Legalize to a VMULL instruction.
9587 SDLoc DL(Op);
9588 SDValue Op0;
9589 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
9590 if (!isMLA) {
9591 Op0 = SkipExtensionForVMULL(N0, DAG);
9592 assert(Op0.getValueType().is64BitVector() &&
9593 Op1.getValueType().is64BitVector() &&
9594 "unexpected types for extended operands to VMULL");
9595 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
9596 }
9597
9598 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
9599 // isel lowering to take advantage of no-stall back to back vmul + vmla.
9600 // vmull q0, d4, d6
9601 // vmlal q0, d5, d6
9602 // is faster than
9603 // vaddl q0, d4, d5
9604 // vmovl q1, d6
9605 // vmul q0, q0, q1
9606 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
9607 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
9608 EVT Op1VT = Op1.getValueType();
9609 return DAG.getNode(N0->getOpcode(), DL, VT,
9610 DAG.getNode(NewOpc, DL, VT,
9611 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
9612 DAG.getNode(NewOpc, DL, VT,
9613 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
9614 }
9615
LowerSDIV_v4i8(SDValue X,SDValue Y,const SDLoc & dl,SelectionDAG & DAG)9616 static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
9617 SelectionDAG &DAG) {
9618 // TODO: Should this propagate fast-math-flags?
9619
9620 // Convert to float
9621 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
9622 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
9623 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X);
9624 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y);
9625 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X);
9626 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y);
9627 // Get reciprocal estimate.
9628 // float4 recip = vrecpeq_f32(yf);
9629 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9630 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9631 Y);
9632 // Because char has a smaller range than uchar, we can actually get away
9633 // without any newton steps. This requires that we use a weird bias
9634 // of 0xb000, however (again, this has been exhaustively tested).
9635 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
9636 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y);
9637 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X);
9638 Y = DAG.getConstant(0xb000, dl, MVT::v4i32);
9639 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y);
9640 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X);
9641 // Convert back to short.
9642 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X);
9643 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X);
9644 return X;
9645 }
9646
LowerSDIV_v4i16(SDValue N0,SDValue N1,const SDLoc & dl,SelectionDAG & DAG)9647 static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
9648 SelectionDAG &DAG) {
9649 // TODO: Should this propagate fast-math-flags?
9650
9651 SDValue N2;
9652 // Convert to float.
9653 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
9654 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
9655 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0);
9656 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1);
9657 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9658 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9659
9660 // Use reciprocal estimate and one refinement step.
9661 // float4 recip = vrecpeq_f32(yf);
9662 // recip *= vrecpsq_f32(yf, recip);
9663 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9664 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9665 N1);
9666 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9667 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9668 N1, N2);
9669 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9670 // Because short has a smaller range than ushort, we can actually get away
9671 // with only a single newton step. This requires that we use a weird bias
9672 // of 89, however (again, this has been exhaustively tested).
9673 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
9674 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9675 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9676 N1 = DAG.getConstant(0x89, dl, MVT::v4i32);
9677 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9678 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9679 // Convert back to integer and return.
9680 // return vmovn_s32(vcvt_s32_f32(result));
9681 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9682 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9683 return N0;
9684 }
9685
LowerSDIV(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)9686 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
9687 const ARMSubtarget *ST) {
9688 EVT VT = Op.getValueType();
9689 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9690 "unexpected type for custom-lowering ISD::SDIV");
9691
9692 SDLoc dl(Op);
9693 SDValue N0 = Op.getOperand(0);
9694 SDValue N1 = Op.getOperand(1);
9695 SDValue N2, N3;
9696
9697 if (VT == MVT::v8i8) {
9698 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0);
9699 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1);
9700
9701 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9702 DAG.getIntPtrConstant(4, dl));
9703 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9704 DAG.getIntPtrConstant(4, dl));
9705 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9706 DAG.getIntPtrConstant(0, dl));
9707 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9708 DAG.getIntPtrConstant(0, dl));
9709
9710 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16
9711 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16
9712
9713 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9714 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9715
9716 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0);
9717 return N0;
9718 }
9719 return LowerSDIV_v4i16(N0, N1, dl, DAG);
9720 }
9721
LowerUDIV(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)9722 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
9723 const ARMSubtarget *ST) {
9724 // TODO: Should this propagate fast-math-flags?
9725 EVT VT = Op.getValueType();
9726 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9727 "unexpected type for custom-lowering ISD::UDIV");
9728
9729 SDLoc dl(Op);
9730 SDValue N0 = Op.getOperand(0);
9731 SDValue N1 = Op.getOperand(1);
9732 SDValue N2, N3;
9733
9734 if (VT == MVT::v8i8) {
9735 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0);
9736 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1);
9737
9738 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9739 DAG.getIntPtrConstant(4, dl));
9740 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9741 DAG.getIntPtrConstant(4, dl));
9742 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9743 DAG.getIntPtrConstant(0, dl));
9744 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9745 DAG.getIntPtrConstant(0, dl));
9746
9747 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
9748 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16
9749
9750 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9751 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9752
9753 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8,
9754 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl,
9755 MVT::i32),
9756 N0);
9757 return N0;
9758 }
9759
9760 // v4i16 sdiv ... Convert to float.
9761 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
9762 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
9763 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0);
9764 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1);
9765 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9766 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9767
9768 // Use reciprocal estimate and two refinement steps.
9769 // float4 recip = vrecpeq_f32(yf);
9770 // recip *= vrecpsq_f32(yf, recip);
9771 // recip *= vrecpsq_f32(yf, recip);
9772 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9773 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9774 BN1);
9775 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9776 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9777 BN1, N2);
9778 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9779 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9780 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9781 BN1, N2);
9782 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9783 // Simply multiplying by the reciprocal estimate can leave us a few ulps
9784 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
9785 // and that it will never cause us to return an answer too large).
9786 // float4 result = as_float4(as_int4(xf*recip) + 2);
9787 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9788 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9789 N1 = DAG.getConstant(2, dl, MVT::v4i32);
9790 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9791 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9792 // Convert back to integer and return.
9793 // return vmovn_u32(vcvt_s32_f32(result));
9794 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9795 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9796 return N0;
9797 }
9798
LowerADDSUBCARRY(SDValue Op,SelectionDAG & DAG)9799 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
9800 SDNode *N = Op.getNode();
9801 EVT VT = N->getValueType(0);
9802 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
9803
9804 SDValue Carry = Op.getOperand(2);
9805
9806 SDLoc DL(Op);
9807
9808 SDValue Result;
9809 if (Op.getOpcode() == ISD::ADDCARRY) {
9810 // This converts the boolean value carry into the carry flag.
9811 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
9812
9813 // Do the addition proper using the carry flag we wanted.
9814 Result = DAG.getNode(ARMISD::ADDE, DL, VTs, Op.getOperand(0),
9815 Op.getOperand(1), Carry);
9816
9817 // Now convert the carry flag into a boolean value.
9818 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
9819 } else {
9820 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we
9821 // have to invert the carry first.
9822 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
9823 DAG.getConstant(1, DL, MVT::i32), Carry);
9824 // This converts the boolean value carry into the carry flag.
9825 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
9826
9827 // Do the subtraction proper using the carry flag we wanted.
9828 Result = DAG.getNode(ARMISD::SUBE, DL, VTs, Op.getOperand(0),
9829 Op.getOperand(1), Carry);
9830
9831 // Now convert the carry flag into a boolean value.
9832 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
9833 // But the carry returned by ARMISD::SUBE is not a borrow as expected
9834 // by ISD::SUBCARRY, so compute 1 - C.
9835 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
9836 DAG.getConstant(1, DL, MVT::i32), Carry);
9837 }
9838
9839 // Return both values.
9840 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry);
9841 }
9842
LowerFSINCOS(SDValue Op,SelectionDAG & DAG) const9843 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
9844 assert(Subtarget->isTargetDarwin());
9845
9846 // For iOS, we want to call an alternative entry point: __sincos_stret,
9847 // return values are passed via sret.
9848 SDLoc dl(Op);
9849 SDValue Arg = Op.getOperand(0);
9850 EVT ArgVT = Arg.getValueType();
9851 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
9852 auto PtrVT = getPointerTy(DAG.getDataLayout());
9853
9854 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
9855 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9856
9857 // Pair of floats / doubles used to pass the result.
9858 Type *RetTy = StructType::get(ArgTy, ArgTy);
9859 auto &DL = DAG.getDataLayout();
9860
9861 ArgListTy Args;
9862 bool ShouldUseSRet = Subtarget->isAPCS_ABI();
9863 SDValue SRet;
9864 if (ShouldUseSRet) {
9865 // Create stack object for sret.
9866 const uint64_t ByteSize = DL.getTypeAllocSize(RetTy);
9867 const Align StackAlign = DL.getPrefTypeAlign(RetTy);
9868 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
9869 SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL));
9870
9871 ArgListEntry Entry;
9872 Entry.Node = SRet;
9873 Entry.Ty = RetTy->getPointerTo();
9874 Entry.IsSExt = false;
9875 Entry.IsZExt = false;
9876 Entry.IsSRet = true;
9877 Args.push_back(Entry);
9878 RetTy = Type::getVoidTy(*DAG.getContext());
9879 }
9880
9881 ArgListEntry Entry;
9882 Entry.Node = Arg;
9883 Entry.Ty = ArgTy;
9884 Entry.IsSExt = false;
9885 Entry.IsZExt = false;
9886 Args.push_back(Entry);
9887
9888 RTLIB::Libcall LC =
9889 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9890 const char *LibcallName = getLibcallName(LC);
9891 CallingConv::ID CC = getLibcallCallingConv(LC);
9892 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
9893
9894 TargetLowering::CallLoweringInfo CLI(DAG);
9895 CLI.setDebugLoc(dl)
9896 .setChain(DAG.getEntryNode())
9897 .setCallee(CC, RetTy, Callee, std::move(Args))
9898 .setDiscardResult(ShouldUseSRet);
9899 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
9900
9901 if (!ShouldUseSRet)
9902 return CallResult.first;
9903
9904 SDValue LoadSin =
9905 DAG.getLoad(ArgVT, dl, CallResult.second, SRet, MachinePointerInfo());
9906
9907 // Address of cos field.
9908 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
9909 DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl));
9910 SDValue LoadCos =
9911 DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add, MachinePointerInfo());
9912
9913 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
9914 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys,
9915 LoadSin.getValue(0), LoadCos.getValue(0));
9916 }
9917
LowerWindowsDIVLibCall(SDValue Op,SelectionDAG & DAG,bool Signed,SDValue & Chain) const9918 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
9919 bool Signed,
9920 SDValue &Chain) const {
9921 EVT VT = Op.getValueType();
9922 assert((VT == MVT::i32 || VT == MVT::i64) &&
9923 "unexpected type for custom lowering DIV");
9924 SDLoc dl(Op);
9925
9926 const auto &DL = DAG.getDataLayout();
9927 const auto &TLI = DAG.getTargetLoweringInfo();
9928
9929 const char *Name = nullptr;
9930 if (Signed)
9931 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64";
9932 else
9933 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64";
9934
9935 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
9936
9937 ARMTargetLowering::ArgListTy Args;
9938
9939 for (auto AI : {1, 0}) {
9940 ArgListEntry Arg;
9941 Arg.Node = Op.getOperand(AI);
9942 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext());
9943 Args.push_back(Arg);
9944 }
9945
9946 CallLoweringInfo CLI(DAG);
9947 CLI.setDebugLoc(dl)
9948 .setChain(Chain)
9949 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()),
9950 ES, std::move(Args));
9951
9952 return LowerCallTo(CLI).first;
9953 }
9954
9955 // This is a code size optimisation: return the original SDIV node to
9956 // DAGCombiner when we don't want to expand SDIV into a sequence of
9957 // instructions, and an empty node otherwise which will cause the
9958 // SDIV to be expanded in DAGCombine.
9959 SDValue
BuildSDIVPow2(SDNode * N,const APInt & Divisor,SelectionDAG & DAG,SmallVectorImpl<SDNode * > & Created) const9960 ARMTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
9961 SelectionDAG &DAG,
9962 SmallVectorImpl<SDNode *> &Created) const {
9963 // TODO: Support SREM
9964 if (N->getOpcode() != ISD::SDIV)
9965 return SDValue();
9966
9967 const auto &ST = DAG.getSubtarget<ARMSubtarget>();
9968 const bool MinSize = ST.hasMinSize();
9969 const bool HasDivide = ST.isThumb() ? ST.hasDivideInThumbMode()
9970 : ST.hasDivideInARMMode();
9971
9972 // Don't touch vector types; rewriting this may lead to scalarizing
9973 // the int divs.
9974 if (N->getOperand(0).getValueType().isVector())
9975 return SDValue();
9976
9977 // Bail if MinSize is not set, and also for both ARM and Thumb mode we need
9978 // hwdiv support for this to be really profitable.
9979 if (!(MinSize && HasDivide))
9980 return SDValue();
9981
9982 // ARM mode is a bit simpler than Thumb: we can handle large power
9983 // of 2 immediates with 1 mov instruction; no further checks required,
9984 // just return the sdiv node.
9985 if (!ST.isThumb())
9986 return SDValue(N, 0);
9987
9988 // In Thumb mode, immediates larger than 128 need a wide 4-byte MOV,
9989 // and thus lose the code size benefits of a MOVS that requires only 2.
9990 // TargetTransformInfo and 'getIntImmCodeSizeCost' could be helpful here,
9991 // but as it's doing exactly this, it's not worth the trouble to get TTI.
9992 if (Divisor.sgt(128))
9993 return SDValue();
9994
9995 return SDValue(N, 0);
9996 }
9997
LowerDIV_Windows(SDValue Op,SelectionDAG & DAG,bool Signed) const9998 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
9999 bool Signed) const {
10000 assert(Op.getValueType() == MVT::i32 &&
10001 "unexpected type for custom lowering DIV");
10002 SDLoc dl(Op);
10003
10004 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
10005 DAG.getEntryNode(), Op.getOperand(1));
10006
10007 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
10008 }
10009
WinDBZCheckDenominator(SelectionDAG & DAG,SDNode * N,SDValue InChain)10010 static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
10011 SDLoc DL(N);
10012 SDValue Op = N->getOperand(1);
10013 if (N->getValueType(0) == MVT::i32)
10014 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, Op);
10015 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
10016 DAG.getConstant(0, DL, MVT::i32));
10017 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
10018 DAG.getConstant(1, DL, MVT::i32));
10019 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain,
10020 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi));
10021 }
10022
ExpandDIV_Windows(SDValue Op,SelectionDAG & DAG,bool Signed,SmallVectorImpl<SDValue> & Results) const10023 void ARMTargetLowering::ExpandDIV_Windows(
10024 SDValue Op, SelectionDAG &DAG, bool Signed,
10025 SmallVectorImpl<SDValue> &Results) const {
10026 const auto &DL = DAG.getDataLayout();
10027 const auto &TLI = DAG.getTargetLoweringInfo();
10028
10029 assert(Op.getValueType() == MVT::i64 &&
10030 "unexpected type for custom lowering DIV");
10031 SDLoc dl(Op);
10032
10033 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode());
10034
10035 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
10036
10037 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
10038 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
10039 DAG.getConstant(32, dl, TLI.getPointerTy(DL)));
10040 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper);
10041
10042 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lower, Upper));
10043 }
10044
LowerPredicateLoad(SDValue Op,SelectionDAG & DAG)10045 static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
10046 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
10047 EVT MemVT = LD->getMemoryVT();
10048 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10049 MemVT == MVT::v16i1) &&
10050 "Expected a predicate type!");
10051 assert(MemVT == Op.getValueType());
10052 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
10053 "Expected a non-extending load");
10054 assert(LD->isUnindexed() && "Expected a unindexed load");
10055
10056 // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit
10057 // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We
10058 // need to make sure that 8/4/2 bits are actually loaded into the correct
10059 // place, which means loading the value and then shuffling the values into
10060 // the bottom bits of the predicate.
10061 // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect
10062 // for BE).
10063 // Speaking of BE, apparently the rest of llvm will assume a reverse order to
10064 // a natural VMSR(load), so needs to be reversed.
10065
10066 SDLoc dl(Op);
10067 SDValue Load = DAG.getExtLoad(
10068 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10069 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
10070 LD->getMemOperand());
10071 SDValue Val = Load;
10072 if (DAG.getDataLayout().isBigEndian())
10073 Val = DAG.getNode(ISD::SRL, dl, MVT::i32,
10074 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Load),
10075 DAG.getConstant(32 - MemVT.getSizeInBits(), dl, MVT::i32));
10076 SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Val);
10077 if (MemVT != MVT::v16i1)
10078 Pred = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MemVT, Pred,
10079 DAG.getConstant(0, dl, MVT::i32));
10080 return DAG.getMergeValues({Pred, Load.getValue(1)}, dl);
10081 }
10082
LowerLOAD(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG) const10083 void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
10084 SelectionDAG &DAG) const {
10085 LoadSDNode *LD = cast<LoadSDNode>(N);
10086 EVT MemVT = LD->getMemoryVT();
10087 assert(LD->isUnindexed() && "Loads should be unindexed at this point.");
10088
10089 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10090 !Subtarget->isThumb1Only() && LD->isVolatile()) {
10091 SDLoc dl(N);
10092 SDValue Result = DAG.getMemIntrinsicNode(
10093 ARMISD::LDRD, dl, DAG.getVTList({MVT::i32, MVT::i32, MVT::Other}),
10094 {LD->getChain(), LD->getBasePtr()}, MemVT, LD->getMemOperand());
10095 SDValue Lo = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 0 : 1);
10096 SDValue Hi = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 1 : 0);
10097 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
10098 Results.append({Pair, Result.getValue(2)});
10099 }
10100 }
10101
LowerPredicateStore(SDValue Op,SelectionDAG & DAG)10102 static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
10103 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
10104 EVT MemVT = ST->getMemoryVT();
10105 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10106 MemVT == MVT::v16i1) &&
10107 "Expected a predicate type!");
10108 assert(MemVT == ST->getValue().getValueType());
10109 assert(!ST->isTruncatingStore() && "Expected a non-extending store");
10110 assert(ST->isUnindexed() && "Expected a unindexed store");
10111
10112 // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with
10113 // top bits unset and a scalar store.
10114 SDLoc dl(Op);
10115 SDValue Build = ST->getValue();
10116 if (MemVT != MVT::v16i1) {
10117 SmallVector<SDValue, 16> Ops;
10118 for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) {
10119 unsigned Elt = DAG.getDataLayout().isBigEndian()
10120 ? MemVT.getVectorNumElements() - I - 1
10121 : I;
10122 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Build,
10123 DAG.getConstant(Elt, dl, MVT::i32)));
10124 }
10125 for (unsigned I = MemVT.getVectorNumElements(); I < 16; I++)
10126 Ops.push_back(DAG.getUNDEF(MVT::i32));
10127 Build = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i1, Ops);
10128 }
10129 SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
10130 if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian())
10131 GRP = DAG.getNode(ISD::SRL, dl, MVT::i32,
10132 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, GRP),
10133 DAG.getConstant(16, dl, MVT::i32));
10134 return DAG.getTruncStore(
10135 ST->getChain(), dl, GRP, ST->getBasePtr(),
10136 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
10137 ST->getMemOperand());
10138 }
10139
LowerSTORE(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)10140 static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG,
10141 const ARMSubtarget *Subtarget) {
10142 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
10143 EVT MemVT = ST->getMemoryVT();
10144 assert(ST->isUnindexed() && "Stores should be unindexed at this point.");
10145
10146 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10147 !Subtarget->isThumb1Only() && ST->isVolatile()) {
10148 SDNode *N = Op.getNode();
10149 SDLoc dl(N);
10150
10151 SDValue Lo = DAG.getNode(
10152 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(),
10153 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 0 : 1, dl,
10154 MVT::i32));
10155 SDValue Hi = DAG.getNode(
10156 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(),
10157 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 1 : 0, dl,
10158 MVT::i32));
10159
10160 return DAG.getMemIntrinsicNode(ARMISD::STRD, dl, DAG.getVTList(MVT::Other),
10161 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10162 MemVT, ST->getMemOperand());
10163 } else if (Subtarget->hasMVEIntegerOps() &&
10164 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10165 MemVT == MVT::v16i1))) {
10166 return LowerPredicateStore(Op, DAG);
10167 }
10168
10169 return SDValue();
10170 }
10171
isZeroVector(SDValue N)10172 static bool isZeroVector(SDValue N) {
10173 return (ISD::isBuildVectorAllZeros(N.getNode()) ||
10174 (N->getOpcode() == ARMISD::VMOVIMM &&
10175 isNullConstant(N->getOperand(0))));
10176 }
10177
LowerMLOAD(SDValue Op,SelectionDAG & DAG)10178 static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
10179 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
10180 MVT VT = Op.getSimpleValueType();
10181 SDValue Mask = N->getMask();
10182 SDValue PassThru = N->getPassThru();
10183 SDLoc dl(Op);
10184
10185 if (isZeroVector(PassThru))
10186 return Op;
10187
10188 // MVE Masked loads use zero as the passthru value. Here we convert undef to
10189 // zero too, and other values are lowered to a select.
10190 SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
10191 DAG.getTargetConstant(0, dl, MVT::i32));
10192 SDValue NewLoad = DAG.getMaskedLoad(
10193 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, ZeroVec,
10194 N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
10195 N->getExtensionType(), N->isExpandingLoad());
10196 SDValue Combo = NewLoad;
10197 bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST ||
10198 PassThru.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
10199 isZeroVector(PassThru->getOperand(0));
10200 if (!PassThru.isUndef() && !PassThruIsCastZero)
10201 Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
10202 return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
10203 }
10204
LowerVecReduce(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)10205 static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
10206 const ARMSubtarget *ST) {
10207 if (!ST->hasMVEIntegerOps())
10208 return SDValue();
10209
10210 SDLoc dl(Op);
10211 unsigned BaseOpcode = 0;
10212 switch (Op->getOpcode()) {
10213 default: llvm_unreachable("Expected VECREDUCE opcode");
10214 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
10215 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
10216 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break;
10217 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break;
10218 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break;
10219 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break;
10220 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break;
10221 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break;
10222 }
10223
10224 SDValue Op0 = Op->getOperand(0);
10225 EVT VT = Op0.getValueType();
10226 EVT EltVT = VT.getVectorElementType();
10227 unsigned NumElts = VT.getVectorNumElements();
10228 unsigned NumActiveLanes = NumElts;
10229
10230 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10231 NumActiveLanes == 2) &&
10232 "Only expected a power 2 vector size");
10233
10234 // Use Mul(X, Rev(X)) until 4 items remain. Going down to 4 vector elements
10235 // allows us to easily extract vector elements from the lanes.
10236 while (NumActiveLanes > 4) {
10237 unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
10238 SDValue Rev = DAG.getNode(RevOpcode, dl, VT, Op0);
10239 Op0 = DAG.getNode(BaseOpcode, dl, VT, Op0, Rev);
10240 NumActiveLanes /= 2;
10241 }
10242
10243 SDValue Res;
10244 if (NumActiveLanes == 4) {
10245 // The remaining 4 elements are summed sequentially
10246 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10247 DAG.getConstant(0 * NumElts / 4, dl, MVT::i32));
10248 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10249 DAG.getConstant(1 * NumElts / 4, dl, MVT::i32));
10250 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10251 DAG.getConstant(2 * NumElts / 4, dl, MVT::i32));
10252 SDValue Ext3 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10253 DAG.getConstant(3 * NumElts / 4, dl, MVT::i32));
10254 SDValue Res0 = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
10255 SDValue Res1 = DAG.getNode(BaseOpcode, dl, EltVT, Ext2, Ext3, Op->getFlags());
10256 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res0, Res1, Op->getFlags());
10257 } else {
10258 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10259 DAG.getConstant(0, dl, MVT::i32));
10260 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10261 DAG.getConstant(1, dl, MVT::i32));
10262 Res = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
10263 }
10264
10265 // Result type may be wider than element type.
10266 if (EltVT != Op->getValueType(0))
10267 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Op->getValueType(0), Res);
10268 return Res;
10269 }
10270
LowerVecReduceF(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST)10271 static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
10272 const ARMSubtarget *ST) {
10273 if (!ST->hasMVEFloatOps())
10274 return SDValue();
10275 return LowerVecReduce(Op, DAG, ST);
10276 }
10277
LowerAtomicLoadStore(SDValue Op,SelectionDAG & DAG)10278 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
10279 if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getSuccessOrdering()))
10280 // Acquire/Release load/store is not legal for targets without a dmb or
10281 // equivalent available.
10282 return SDValue();
10283
10284 // Monotonic load/store is legal for all targets.
10285 return Op;
10286 }
10287
ReplaceREADCYCLECOUNTER(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG,const ARMSubtarget * Subtarget)10288 static void ReplaceREADCYCLECOUNTER(SDNode *N,
10289 SmallVectorImpl<SDValue> &Results,
10290 SelectionDAG &DAG,
10291 const ARMSubtarget *Subtarget) {
10292 SDLoc DL(N);
10293 // Under Power Management extensions, the cycle-count is:
10294 // mrc p15, #0, <Rt>, c9, c13, #0
10295 SDValue Ops[] = { N->getOperand(0), // Chain
10296 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
10297 DAG.getTargetConstant(15, DL, MVT::i32),
10298 DAG.getTargetConstant(0, DL, MVT::i32),
10299 DAG.getTargetConstant(9, DL, MVT::i32),
10300 DAG.getTargetConstant(13, DL, MVT::i32),
10301 DAG.getTargetConstant(0, DL, MVT::i32)
10302 };
10303
10304 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
10305 DAG.getVTList(MVT::i32, MVT::Other), Ops);
10306 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32,
10307 DAG.getConstant(0, DL, MVT::i32)));
10308 Results.push_back(Cycles32.getValue(1));
10309 }
10310
createGPRPairNode(SelectionDAG & DAG,SDValue V)10311 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
10312 SDLoc dl(V.getNode());
10313 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i32);
10314 SDValue VHi = DAG.getAnyExtOrTrunc(
10315 DAG.getNode(ISD::SRL, dl, MVT::i64, V, DAG.getConstant(32, dl, MVT::i32)),
10316 dl, MVT::i32);
10317 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10318 if (isBigEndian)
10319 std::swap (VLo, VHi);
10320 SDValue RegClass =
10321 DAG.getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
10322 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32);
10323 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32);
10324 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
10325 return SDValue(
10326 DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10327 }
10328
ReplaceCMP_SWAP_64Results(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG)10329 static void ReplaceCMP_SWAP_64Results(SDNode *N,
10330 SmallVectorImpl<SDValue> & Results,
10331 SelectionDAG &DAG) {
10332 assert(N->getValueType(0) == MVT::i64 &&
10333 "AtomicCmpSwap on types less than 64 should be legal");
10334 SDValue Ops[] = {N->getOperand(1),
10335 createGPRPairNode(DAG, N->getOperand(2)),
10336 createGPRPairNode(DAG, N->getOperand(3)),
10337 N->getOperand(0)};
10338 SDNode *CmpSwap = DAG.getMachineNode(
10339 ARM::CMP_SWAP_64, SDLoc(N),
10340 DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
10341
10342 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
10343 DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
10344
10345 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10346
10347 SDValue Lo =
10348 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_1 : ARM::gsub_0,
10349 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
10350 SDValue Hi =
10351 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_0 : ARM::gsub_1,
10352 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
10353 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i64, Lo, Hi));
10354 Results.push_back(SDValue(CmpSwap, 2));
10355 }
10356
LowerFSETCC(SDValue Op,SelectionDAG & DAG) const10357 SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
10358 SDLoc dl(Op);
10359 EVT VT = Op.getValueType();
10360 SDValue Chain = Op.getOperand(0);
10361 SDValue LHS = Op.getOperand(1);
10362 SDValue RHS = Op.getOperand(2);
10363 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get();
10364 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
10365
10366 // If we don't have instructions of this float type then soften to a libcall
10367 // and use SETCC instead.
10368 if (isUnsupportedFloatingType(LHS.getValueType())) {
10369 DAG.getTargetLoweringInfo().softenSetCCOperands(
10370 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS, Chain, IsSignaling);
10371 if (!RHS.getNode()) {
10372 RHS = DAG.getConstant(0, dl, LHS.getValueType());
10373 CC = ISD::SETNE;
10374 }
10375 SDValue Result = DAG.getNode(ISD::SETCC, dl, VT, LHS, RHS,
10376 DAG.getCondCode(CC));
10377 return DAG.getMergeValues({Result, Chain}, dl);
10378 }
10379
10380 ARMCC::CondCodes CondCode, CondCode2;
10381 FPCCToARMCC(CC, CondCode, CondCode2);
10382
10383 // FIXME: Chain is not handled correctly here. Currently the FPSCR is implicit
10384 // in CMPFP and CMPFPE, but instead it should be made explicit by these
10385 // instructions using a chain instead of glue. This would also fix the problem
10386 // here (and also in LowerSELECT_CC) where we generate two comparisons when
10387 // CondCode2 != AL.
10388 SDValue True = DAG.getConstant(1, dl, VT);
10389 SDValue False = DAG.getConstant(0, dl, VT);
10390 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
10391 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
10392 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10393 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10394 if (CondCode2 != ARMCC::AL) {
10395 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
10396 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10397 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10398 }
10399 return DAG.getMergeValues({Result, Chain}, dl);
10400 }
10401
LowerSPONENTRY(SDValue Op,SelectionDAG & DAG) const10402 SDValue ARMTargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
10403 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
10404
10405 EVT VT = getPointerTy(DAG.getDataLayout());
10406 SDLoc DL(Op);
10407 int FI = MFI.CreateFixedObject(4, 0, false);
10408 return DAG.getFrameIndex(FI, VT);
10409 }
10410
LowerOperation(SDValue Op,SelectionDAG & DAG) const10411 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10412 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump());
10413 switch (Op.getOpcode()) {
10414 default: llvm_unreachable("Don't know how to custom lower this!");
10415 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
10416 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
10417 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
10418 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
10419 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
10420 case ISD::SELECT: return LowerSELECT(Op, DAG);
10421 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
10422 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
10423 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
10424 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
10425 case ISD::VASTART: return LowerVASTART(Op, DAG);
10426 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
10427 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
10428 case ISD::SINT_TO_FP:
10429 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
10430 case ISD::STRICT_FP_TO_SINT:
10431 case ISD::STRICT_FP_TO_UINT:
10432 case ISD::FP_TO_SINT:
10433 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
10434 case ISD::FP_TO_SINT_SAT:
10435 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG, Subtarget);
10436 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
10437 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
10438 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
10439 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
10440 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
10441 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
10442 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget);
10443 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
10444 Subtarget);
10445 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG, Subtarget);
10446 case ISD::SHL:
10447 case ISD::SRL:
10448 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget);
10449 case ISD::SREM: return LowerREM(Op.getNode(), DAG);
10450 case ISD::UREM: return LowerREM(Op.getNode(), DAG);
10451 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
10452 case ISD::SRL_PARTS:
10453 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
10454 case ISD::CTTZ:
10455 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
10456 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget);
10457 case ISD::SETCC: return LowerVSETCC(Op, DAG, Subtarget);
10458 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
10459 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget);
10460 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget);
10461 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
10462 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, Subtarget);
10463 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
10464 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, Subtarget);
10465 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, Subtarget);
10466 case ISD::TRUNCATE: return LowerTruncate(Op.getNode(), DAG, Subtarget);
10467 case ISD::SIGN_EXTEND:
10468 case ISD::ZERO_EXTEND: return LowerVectorExtend(Op.getNode(), DAG, Subtarget);
10469 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
10470 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
10471 case ISD::MUL: return LowerMUL(Op, DAG);
10472 case ISD::SDIV:
10473 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector())
10474 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
10475 return LowerSDIV(Op, DAG, Subtarget);
10476 case ISD::UDIV:
10477 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector())
10478 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
10479 return LowerUDIV(Op, DAG, Subtarget);
10480 case ISD::ADDCARRY:
10481 case ISD::SUBCARRY: return LowerADDSUBCARRY(Op, DAG);
10482 case ISD::SADDO:
10483 case ISD::SSUBO:
10484 return LowerSignedALUO(Op, DAG);
10485 case ISD::UADDO:
10486 case ISD::USUBO:
10487 return LowerUnsignedALUO(Op, DAG);
10488 case ISD::SADDSAT:
10489 case ISD::SSUBSAT:
10490 case ISD::UADDSAT:
10491 case ISD::USUBSAT:
10492 return LowerADDSUBSAT(Op, DAG, Subtarget);
10493 case ISD::LOAD:
10494 return LowerPredicateLoad(Op, DAG);
10495 case ISD::STORE:
10496 return LowerSTORE(Op, DAG, Subtarget);
10497 case ISD::MLOAD:
10498 return LowerMLOAD(Op, DAG);
10499 case ISD::VECREDUCE_MUL:
10500 case ISD::VECREDUCE_AND:
10501 case ISD::VECREDUCE_OR:
10502 case ISD::VECREDUCE_XOR:
10503 return LowerVecReduce(Op, DAG, Subtarget);
10504 case ISD::VECREDUCE_FADD:
10505 case ISD::VECREDUCE_FMUL:
10506 case ISD::VECREDUCE_FMIN:
10507 case ISD::VECREDUCE_FMAX:
10508 return LowerVecReduceF(Op, DAG, Subtarget);
10509 case ISD::ATOMIC_LOAD:
10510 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG);
10511 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG);
10512 case ISD::SDIVREM:
10513 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
10514 case ISD::DYNAMIC_STACKALLOC:
10515 if (Subtarget->isTargetWindows())
10516 return LowerDYNAMIC_STACKALLOC(Op, DAG);
10517 llvm_unreachable("Don't know how to custom lower this!");
10518 case ISD::STRICT_FP_ROUND:
10519 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
10520 case ISD::STRICT_FP_EXTEND:
10521 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
10522 case ISD::STRICT_FSETCC:
10523 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG);
10524 case ISD::SPONENTRY:
10525 return LowerSPONENTRY(Op, DAG);
10526 case ARMISD::WIN__DBZCHK: return SDValue();
10527 }
10528 }
10529
ReplaceLongIntrinsic(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG)10530 static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results,
10531 SelectionDAG &DAG) {
10532 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
10533 unsigned Opc = 0;
10534 if (IntNo == Intrinsic::arm_smlald)
10535 Opc = ARMISD::SMLALD;
10536 else if (IntNo == Intrinsic::arm_smlaldx)
10537 Opc = ARMISD::SMLALDX;
10538 else if (IntNo == Intrinsic::arm_smlsld)
10539 Opc = ARMISD::SMLSLD;
10540 else if (IntNo == Intrinsic::arm_smlsldx)
10541 Opc = ARMISD::SMLSLDX;
10542 else
10543 return;
10544
10545 SDLoc dl(N);
10546 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
10547 N->getOperand(3),
10548 DAG.getConstant(0, dl, MVT::i32));
10549 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
10550 N->getOperand(3),
10551 DAG.getConstant(1, dl, MVT::i32));
10552
10553 SDValue LongMul = DAG.getNode(Opc, dl,
10554 DAG.getVTList(MVT::i32, MVT::i32),
10555 N->getOperand(1), N->getOperand(2),
10556 Lo, Hi);
10557 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
10558 LongMul.getValue(0), LongMul.getValue(1)));
10559 }
10560
10561 /// ReplaceNodeResults - Replace the results of node with an illegal result
10562 /// type with new values built out of custom code.
ReplaceNodeResults(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG) const10563 void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
10564 SmallVectorImpl<SDValue> &Results,
10565 SelectionDAG &DAG) const {
10566 SDValue Res;
10567 switch (N->getOpcode()) {
10568 default:
10569 llvm_unreachable("Don't know how to custom expand this!");
10570 case ISD::READ_REGISTER:
10571 ExpandREAD_REGISTER(N, Results, DAG);
10572 break;
10573 case ISD::BITCAST:
10574 Res = ExpandBITCAST(N, DAG, Subtarget);
10575 break;
10576 case ISD::SRL:
10577 case ISD::SRA:
10578 case ISD::SHL:
10579 Res = Expand64BitShift(N, DAG, Subtarget);
10580 break;
10581 case ISD::SREM:
10582 case ISD::UREM:
10583 Res = LowerREM(N, DAG);
10584 break;
10585 case ISD::SDIVREM:
10586 case ISD::UDIVREM:
10587 Res = LowerDivRem(SDValue(N, 0), DAG);
10588 assert(Res.getNumOperands() == 2 && "DivRem needs two values");
10589 Results.push_back(Res.getValue(0));
10590 Results.push_back(Res.getValue(1));
10591 return;
10592 case ISD::SADDSAT:
10593 case ISD::SSUBSAT:
10594 case ISD::UADDSAT:
10595 case ISD::USUBSAT:
10596 Res = LowerADDSUBSAT(SDValue(N, 0), DAG, Subtarget);
10597 break;
10598 case ISD::READCYCLECOUNTER:
10599 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
10600 return;
10601 case ISD::UDIV:
10602 case ISD::SDIV:
10603 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows");
10604 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV,
10605 Results);
10606 case ISD::ATOMIC_CMP_SWAP:
10607 ReplaceCMP_SWAP_64Results(N, Results, DAG);
10608 return;
10609 case ISD::INTRINSIC_WO_CHAIN:
10610 return ReplaceLongIntrinsic(N, Results, DAG);
10611 case ISD::LOAD:
10612 LowerLOAD(N, Results, DAG);
10613 break;
10614 case ISD::TRUNCATE:
10615 Res = LowerTruncate(N, DAG, Subtarget);
10616 break;
10617 case ISD::SIGN_EXTEND:
10618 case ISD::ZERO_EXTEND:
10619 Res = LowerVectorExtend(N, DAG, Subtarget);
10620 break;
10621 case ISD::FP_TO_SINT_SAT:
10622 case ISD::FP_TO_UINT_SAT:
10623 Res = LowerFP_TO_INT_SAT(SDValue(N, 0), DAG, Subtarget);
10624 break;
10625 }
10626 if (Res.getNode())
10627 Results.push_back(Res);
10628 }
10629
10630 //===----------------------------------------------------------------------===//
10631 // ARM Scheduler Hooks
10632 //===----------------------------------------------------------------------===//
10633
10634 /// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
10635 /// registers the function context.
SetupEntryBlockForSjLj(MachineInstr & MI,MachineBasicBlock * MBB,MachineBasicBlock * DispatchBB,int FI) const10636 void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
10637 MachineBasicBlock *MBB,
10638 MachineBasicBlock *DispatchBB,
10639 int FI) const {
10640 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
10641 "ROPI/RWPI not currently supported with SjLj");
10642 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10643 DebugLoc dl = MI.getDebugLoc();
10644 MachineFunction *MF = MBB->getParent();
10645 MachineRegisterInfo *MRI = &MF->getRegInfo();
10646 MachineConstantPool *MCP = MF->getConstantPool();
10647 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
10648 const Function &F = MF->getFunction();
10649
10650 bool isThumb = Subtarget->isThumb();
10651 bool isThumb2 = Subtarget->isThumb2();
10652
10653 unsigned PCLabelId = AFI->createPICLabelUId();
10654 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
10655 ARMConstantPoolValue *CPV =
10656 ARMConstantPoolMBB::Create(F.getContext(), DispatchBB, PCLabelId, PCAdj);
10657 unsigned CPI = MCP->getConstantPoolIndex(CPV, Align(4));
10658
10659 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
10660 : &ARM::GPRRegClass;
10661
10662 // Grab constant pool and fixed stack memory operands.
10663 MachineMemOperand *CPMMO =
10664 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
10665 MachineMemOperand::MOLoad, 4, Align(4));
10666
10667 MachineMemOperand *FIMMOSt =
10668 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
10669 MachineMemOperand::MOStore, 4, Align(4));
10670
10671 // Load the address of the dispatch MBB into the jump buffer.
10672 if (isThumb2) {
10673 // Incoming value: jbuf
10674 // ldr.n r5, LCPI1_1
10675 // orr r5, r5, #1
10676 // add r5, pc
10677 // str r5, [$jbuf, #+4] ; &jbuf[1]
10678 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10679 BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1)
10680 .addConstantPoolIndex(CPI)
10681 .addMemOperand(CPMMO)
10682 .add(predOps(ARMCC::AL));
10683 // Set the low bit because of thumb mode.
10684 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10685 BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2)
10686 .addReg(NewVReg1, RegState::Kill)
10687 .addImm(0x01)
10688 .add(predOps(ARMCC::AL))
10689 .add(condCodeOp());
10690 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10691 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3)
10692 .addReg(NewVReg2, RegState::Kill)
10693 .addImm(PCLabelId);
10694 BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12))
10695 .addReg(NewVReg3, RegState::Kill)
10696 .addFrameIndex(FI)
10697 .addImm(36) // &jbuf[1] :: pc
10698 .addMemOperand(FIMMOSt)
10699 .add(predOps(ARMCC::AL));
10700 } else if (isThumb) {
10701 // Incoming value: jbuf
10702 // ldr.n r1, LCPI1_4
10703 // add r1, pc
10704 // mov r2, #1
10705 // orrs r1, r2
10706 // add r2, $jbuf, #+4 ; &jbuf[1]
10707 // str r1, [r2]
10708 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10709 BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1)
10710 .addConstantPoolIndex(CPI)
10711 .addMemOperand(CPMMO)
10712 .add(predOps(ARMCC::AL));
10713 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10714 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2)
10715 .addReg(NewVReg1, RegState::Kill)
10716 .addImm(PCLabelId);
10717 // Set the low bit because of thumb mode.
10718 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10719 BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3)
10720 .addReg(ARM::CPSR, RegState::Define)
10721 .addImm(1)
10722 .add(predOps(ARMCC::AL));
10723 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10724 BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4)
10725 .addReg(ARM::CPSR, RegState::Define)
10726 .addReg(NewVReg2, RegState::Kill)
10727 .addReg(NewVReg3, RegState::Kill)
10728 .add(predOps(ARMCC::AL));
10729 Register NewVReg5 = MRI->createVirtualRegister(TRC);
10730 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5)
10731 .addFrameIndex(FI)
10732 .addImm(36); // &jbuf[1] :: pc
10733 BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi))
10734 .addReg(NewVReg4, RegState::Kill)
10735 .addReg(NewVReg5, RegState::Kill)
10736 .addImm(0)
10737 .addMemOperand(FIMMOSt)
10738 .add(predOps(ARMCC::AL));
10739 } else {
10740 // Incoming value: jbuf
10741 // ldr r1, LCPI1_1
10742 // add r1, pc, r1
10743 // str r1, [$jbuf, #+4] ; &jbuf[1]
10744 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10745 BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1)
10746 .addConstantPoolIndex(CPI)
10747 .addImm(0)
10748 .addMemOperand(CPMMO)
10749 .add(predOps(ARMCC::AL));
10750 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10751 BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2)
10752 .addReg(NewVReg1, RegState::Kill)
10753 .addImm(PCLabelId)
10754 .add(predOps(ARMCC::AL));
10755 BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12))
10756 .addReg(NewVReg2, RegState::Kill)
10757 .addFrameIndex(FI)
10758 .addImm(36) // &jbuf[1] :: pc
10759 .addMemOperand(FIMMOSt)
10760 .add(predOps(ARMCC::AL));
10761 }
10762 }
10763
EmitSjLjDispatchBlock(MachineInstr & MI,MachineBasicBlock * MBB) const10764 void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
10765 MachineBasicBlock *MBB) const {
10766 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10767 DebugLoc dl = MI.getDebugLoc();
10768 MachineFunction *MF = MBB->getParent();
10769 MachineRegisterInfo *MRI = &MF->getRegInfo();
10770 MachineFrameInfo &MFI = MF->getFrameInfo();
10771 int FI = MFI.getFunctionContextIndex();
10772
10773 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
10774 : &ARM::GPRnopcRegClass;
10775
10776 // Get a mapping of the call site numbers to all of the landing pads they're
10777 // associated with.
10778 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
10779 unsigned MaxCSNum = 0;
10780 for (MachineBasicBlock &BB : *MF) {
10781 if (!BB.isEHPad())
10782 continue;
10783
10784 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
10785 // pad.
10786 for (MachineInstr &II : BB) {
10787 if (!II.isEHLabel())
10788 continue;
10789
10790 MCSymbol *Sym = II.getOperand(0).getMCSymbol();
10791 if (!MF->hasCallSiteLandingPad(Sym)) continue;
10792
10793 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
10794 for (unsigned Idx : CallSiteIdxs) {
10795 CallSiteNumToLPad[Idx].push_back(&BB);
10796 MaxCSNum = std::max(MaxCSNum, Idx);
10797 }
10798 break;
10799 }
10800 }
10801
10802 // Get an ordered list of the machine basic blocks for the jump table.
10803 std::vector<MachineBasicBlock*> LPadList;
10804 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
10805 LPadList.reserve(CallSiteNumToLPad.size());
10806 for (unsigned I = 1; I <= MaxCSNum; ++I) {
10807 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
10808 for (MachineBasicBlock *MBB : MBBList) {
10809 LPadList.push_back(MBB);
10810 InvokeBBs.insert(MBB->pred_begin(), MBB->pred_end());
10811 }
10812 }
10813
10814 assert(!LPadList.empty() &&
10815 "No landing pad destinations for the dispatch jump table!");
10816
10817 // Create the jump table and associated information.
10818 MachineJumpTableInfo *JTI =
10819 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline);
10820 unsigned MJTI = JTI->createJumpTableIndex(LPadList);
10821
10822 // Create the MBBs for the dispatch code.
10823
10824 // Shove the dispatch's address into the return slot in the function context.
10825 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
10826 DispatchBB->setIsEHPad();
10827
10828 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
10829 unsigned trap_opcode;
10830 if (Subtarget->isThumb())
10831 trap_opcode = ARM::tTRAP;
10832 else
10833 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
10834
10835 BuildMI(TrapBB, dl, TII->get(trap_opcode));
10836 DispatchBB->addSuccessor(TrapBB);
10837
10838 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
10839 DispatchBB->addSuccessor(DispContBB);
10840
10841 // Insert and MBBs.
10842 MF->insert(MF->end(), DispatchBB);
10843 MF->insert(MF->end(), DispContBB);
10844 MF->insert(MF->end(), TrapBB);
10845
10846 // Insert code into the entry block that creates and registers the function
10847 // context.
10848 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
10849
10850 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
10851 MachinePointerInfo::getFixedStack(*MF, FI),
10852 MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, Align(4));
10853
10854 MachineInstrBuilder MIB;
10855 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
10856
10857 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
10858 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
10859
10860 // Add a register mask with no preserved registers. This results in all
10861 // registers being marked as clobbered. This can't work if the dispatch block
10862 // is in a Thumb1 function and is linked with ARM code which uses the FP
10863 // registers, as there is no way to preserve the FP registers in Thumb1 mode.
10864 MIB.addRegMask(RI.getSjLjDispatchPreservedMask(*MF));
10865
10866 bool IsPositionIndependent = isPositionIndependent();
10867 unsigned NumLPads = LPadList.size();
10868 if (Subtarget->isThumb2()) {
10869 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10870 BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1)
10871 .addFrameIndex(FI)
10872 .addImm(4)
10873 .addMemOperand(FIMMOLd)
10874 .add(predOps(ARMCC::AL));
10875
10876 if (NumLPads < 256) {
10877 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri))
10878 .addReg(NewVReg1)
10879 .addImm(LPadList.size())
10880 .add(predOps(ARMCC::AL));
10881 } else {
10882 Register VReg1 = MRI->createVirtualRegister(TRC);
10883 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1)
10884 .addImm(NumLPads & 0xFFFF)
10885 .add(predOps(ARMCC::AL));
10886
10887 unsigned VReg2 = VReg1;
10888 if ((NumLPads & 0xFFFF0000) != 0) {
10889 VReg2 = MRI->createVirtualRegister(TRC);
10890 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2)
10891 .addReg(VReg1)
10892 .addImm(NumLPads >> 16)
10893 .add(predOps(ARMCC::AL));
10894 }
10895
10896 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr))
10897 .addReg(NewVReg1)
10898 .addReg(VReg2)
10899 .add(predOps(ARMCC::AL));
10900 }
10901
10902 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc))
10903 .addMBB(TrapBB)
10904 .addImm(ARMCC::HI)
10905 .addReg(ARM::CPSR);
10906
10907 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10908 BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT), NewVReg3)
10909 .addJumpTableIndex(MJTI)
10910 .add(predOps(ARMCC::AL));
10911
10912 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10913 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4)
10914 .addReg(NewVReg3, RegState::Kill)
10915 .addReg(NewVReg1)
10916 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))
10917 .add(predOps(ARMCC::AL))
10918 .add(condCodeOp());
10919
10920 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT))
10921 .addReg(NewVReg4, RegState::Kill)
10922 .addReg(NewVReg1)
10923 .addJumpTableIndex(MJTI);
10924 } else if (Subtarget->isThumb()) {
10925 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10926 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1)
10927 .addFrameIndex(FI)
10928 .addImm(1)
10929 .addMemOperand(FIMMOLd)
10930 .add(predOps(ARMCC::AL));
10931
10932 if (NumLPads < 256) {
10933 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8))
10934 .addReg(NewVReg1)
10935 .addImm(NumLPads)
10936 .add(predOps(ARMCC::AL));
10937 } else {
10938 MachineConstantPool *ConstantPool = MF->getConstantPool();
10939 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
10940 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
10941
10942 // MachineConstantPool wants an explicit alignment.
10943 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
10944 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10945
10946 Register VReg1 = MRI->createVirtualRegister(TRC);
10947 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci))
10948 .addReg(VReg1, RegState::Define)
10949 .addConstantPoolIndex(Idx)
10950 .add(predOps(ARMCC::AL));
10951 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr))
10952 .addReg(NewVReg1)
10953 .addReg(VReg1)
10954 .add(predOps(ARMCC::AL));
10955 }
10956
10957 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc))
10958 .addMBB(TrapBB)
10959 .addImm(ARMCC::HI)
10960 .addReg(ARM::CPSR);
10961
10962 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10963 BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2)
10964 .addReg(ARM::CPSR, RegState::Define)
10965 .addReg(NewVReg1)
10966 .addImm(2)
10967 .add(predOps(ARMCC::AL));
10968
10969 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10970 BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3)
10971 .addJumpTableIndex(MJTI)
10972 .add(predOps(ARMCC::AL));
10973
10974 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10975 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4)
10976 .addReg(ARM::CPSR, RegState::Define)
10977 .addReg(NewVReg2, RegState::Kill)
10978 .addReg(NewVReg3)
10979 .add(predOps(ARMCC::AL));
10980
10981 MachineMemOperand *JTMMOLd =
10982 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
10983 MachineMemOperand::MOLoad, 4, Align(4));
10984
10985 Register NewVReg5 = MRI->createVirtualRegister(TRC);
10986 BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5)
10987 .addReg(NewVReg4, RegState::Kill)
10988 .addImm(0)
10989 .addMemOperand(JTMMOLd)
10990 .add(predOps(ARMCC::AL));
10991
10992 unsigned NewVReg6 = NewVReg5;
10993 if (IsPositionIndependent) {
10994 NewVReg6 = MRI->createVirtualRegister(TRC);
10995 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6)
10996 .addReg(ARM::CPSR, RegState::Define)
10997 .addReg(NewVReg5, RegState::Kill)
10998 .addReg(NewVReg3)
10999 .add(predOps(ARMCC::AL));
11000 }
11001
11002 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr))
11003 .addReg(NewVReg6, RegState::Kill)
11004 .addJumpTableIndex(MJTI);
11005 } else {
11006 Register NewVReg1 = MRI->createVirtualRegister(TRC);
11007 BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1)
11008 .addFrameIndex(FI)
11009 .addImm(4)
11010 .addMemOperand(FIMMOLd)
11011 .add(predOps(ARMCC::AL));
11012
11013 if (NumLPads < 256) {
11014 BuildMI(DispatchBB, dl, TII->get(ARM::CMPri))
11015 .addReg(NewVReg1)
11016 .addImm(NumLPads)
11017 .add(predOps(ARMCC::AL));
11018 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11019 Register VReg1 = MRI->createVirtualRegister(TRC);
11020 BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1)
11021 .addImm(NumLPads & 0xFFFF)
11022 .add(predOps(ARMCC::AL));
11023
11024 unsigned VReg2 = VReg1;
11025 if ((NumLPads & 0xFFFF0000) != 0) {
11026 VReg2 = MRI->createVirtualRegister(TRC);
11027 BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2)
11028 .addReg(VReg1)
11029 .addImm(NumLPads >> 16)
11030 .add(predOps(ARMCC::AL));
11031 }
11032
11033 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
11034 .addReg(NewVReg1)
11035 .addReg(VReg2)
11036 .add(predOps(ARMCC::AL));
11037 } else {
11038 MachineConstantPool *ConstantPool = MF->getConstantPool();
11039 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
11040 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
11041
11042 // MachineConstantPool wants an explicit alignment.
11043 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11044 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11045
11046 Register VReg1 = MRI->createVirtualRegister(TRC);
11047 BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp))
11048 .addReg(VReg1, RegState::Define)
11049 .addConstantPoolIndex(Idx)
11050 .addImm(0)
11051 .add(predOps(ARMCC::AL));
11052 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
11053 .addReg(NewVReg1)
11054 .addReg(VReg1, RegState::Kill)
11055 .add(predOps(ARMCC::AL));
11056 }
11057
11058 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc))
11059 .addMBB(TrapBB)
11060 .addImm(ARMCC::HI)
11061 .addReg(ARM::CPSR);
11062
11063 Register NewVReg3 = MRI->createVirtualRegister(TRC);
11064 BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3)
11065 .addReg(NewVReg1)
11066 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))
11067 .add(predOps(ARMCC::AL))
11068 .add(condCodeOp());
11069 Register NewVReg4 = MRI->createVirtualRegister(TRC);
11070 BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4)
11071 .addJumpTableIndex(MJTI)
11072 .add(predOps(ARMCC::AL));
11073
11074 MachineMemOperand *JTMMOLd =
11075 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
11076 MachineMemOperand::MOLoad, 4, Align(4));
11077 Register NewVReg5 = MRI->createVirtualRegister(TRC);
11078 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5)
11079 .addReg(NewVReg3, RegState::Kill)
11080 .addReg(NewVReg4)
11081 .addImm(0)
11082 .addMemOperand(JTMMOLd)
11083 .add(predOps(ARMCC::AL));
11084
11085 if (IsPositionIndependent) {
11086 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd))
11087 .addReg(NewVReg5, RegState::Kill)
11088 .addReg(NewVReg4)
11089 .addJumpTableIndex(MJTI);
11090 } else {
11091 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr))
11092 .addReg(NewVReg5, RegState::Kill)
11093 .addJumpTableIndex(MJTI);
11094 }
11095 }
11096
11097 // Add the jump table entries as successors to the MBB.
11098 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
11099 for (MachineBasicBlock *CurMBB : LPadList) {
11100 if (SeenMBBs.insert(CurMBB).second)
11101 DispContBB->addSuccessor(CurMBB);
11102 }
11103
11104 // N.B. the order the invoke BBs are processed in doesn't matter here.
11105 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
11106 SmallVector<MachineBasicBlock*, 64> MBBLPads;
11107 for (MachineBasicBlock *BB : InvokeBBs) {
11108
11109 // Remove the landing pad successor from the invoke block and replace it
11110 // with the new dispatch block.
11111 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors());
11112 while (!Successors.empty()) {
11113 MachineBasicBlock *SMBB = Successors.pop_back_val();
11114 if (SMBB->isEHPad()) {
11115 BB->removeSuccessor(SMBB);
11116 MBBLPads.push_back(SMBB);
11117 }
11118 }
11119
11120 BB->addSuccessor(DispatchBB, BranchProbability::getZero());
11121 BB->normalizeSuccProbs();
11122
11123 // Find the invoke call and mark all of the callee-saved registers as
11124 // 'implicit defined' so that they're spilled. This prevents code from
11125 // moving instructions to before the EH block, where they will never be
11126 // executed.
11127 for (MachineBasicBlock::reverse_iterator
11128 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11129 if (!II->isCall()) continue;
11130
11131 DenseMap<unsigned, bool> DefRegs;
11132 for (MachineInstr::mop_iterator
11133 OI = II->operands_begin(), OE = II->operands_end();
11134 OI != OE; ++OI) {
11135 if (!OI->isReg()) continue;
11136 DefRegs[OI->getReg()] = true;
11137 }
11138
11139 MachineInstrBuilder MIB(*MF, &*II);
11140
11141 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
11142 unsigned Reg = SavedRegs[i];
11143 if (Subtarget->isThumb2() &&
11144 !ARM::tGPRRegClass.contains(Reg) &&
11145 !ARM::hGPRRegClass.contains(Reg))
11146 continue;
11147 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11148 continue;
11149 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11150 continue;
11151 if (!DefRegs[Reg])
11152 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
11153 }
11154
11155 break;
11156 }
11157 }
11158
11159 // Mark all former landing pads as non-landing pads. The dispatch is the only
11160 // landing pad now.
11161 for (MachineBasicBlock *MBBLPad : MBBLPads)
11162 MBBLPad->setIsEHPad(false);
11163
11164 // The instruction is gone now.
11165 MI.eraseFromParent();
11166 }
11167
11168 static
OtherSucc(MachineBasicBlock * MBB,MachineBasicBlock * Succ)11169 MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
11170 for (MachineBasicBlock *S : MBB->successors())
11171 if (S != Succ)
11172 return S;
11173 llvm_unreachable("Expecting a BB with two successors!");
11174 }
11175
11176 /// Return the load opcode for a given load size. If load size >= 8,
11177 /// neon opcode will be returned.
getLdOpcode(unsigned LdSize,bool IsThumb1,bool IsThumb2)11178 static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
11179 if (LdSize >= 8)
11180 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11181 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11182 if (IsThumb1)
11183 return LdSize == 4 ? ARM::tLDRi
11184 : LdSize == 2 ? ARM::tLDRHi
11185 : LdSize == 1 ? ARM::tLDRBi : 0;
11186 if (IsThumb2)
11187 return LdSize == 4 ? ARM::t2LDR_POST
11188 : LdSize == 2 ? ARM::t2LDRH_POST
11189 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11190 return LdSize == 4 ? ARM::LDR_POST_IMM
11191 : LdSize == 2 ? ARM::LDRH_POST
11192 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11193 }
11194
11195 /// Return the store opcode for a given store size. If store size >= 8,
11196 /// neon opcode will be returned.
getStOpcode(unsigned StSize,bool IsThumb1,bool IsThumb2)11197 static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
11198 if (StSize >= 8)
11199 return StSize == 16 ? ARM::VST1q32wb_fixed
11200 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11201 if (IsThumb1)
11202 return StSize == 4 ? ARM::tSTRi
11203 : StSize == 2 ? ARM::tSTRHi
11204 : StSize == 1 ? ARM::tSTRBi : 0;
11205 if (IsThumb2)
11206 return StSize == 4 ? ARM::t2STR_POST
11207 : StSize == 2 ? ARM::t2STRH_POST
11208 : StSize == 1 ? ARM::t2STRB_POST : 0;
11209 return StSize == 4 ? ARM::STR_POST_IMM
11210 : StSize == 2 ? ARM::STRH_POST
11211 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11212 }
11213
11214 /// Emit a post-increment load operation with given size. The instructions
11215 /// will be added to BB at Pos.
emitPostLd(MachineBasicBlock * BB,MachineBasicBlock::iterator Pos,const TargetInstrInfo * TII,const DebugLoc & dl,unsigned LdSize,unsigned Data,unsigned AddrIn,unsigned AddrOut,bool IsThumb1,bool IsThumb2)11216 static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11217 const TargetInstrInfo *TII, const DebugLoc &dl,
11218 unsigned LdSize, unsigned Data, unsigned AddrIn,
11219 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11220 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
11221 assert(LdOpc != 0 && "Should have a load opcode");
11222 if (LdSize >= 8) {
11223 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11224 .addReg(AddrOut, RegState::Define)
11225 .addReg(AddrIn)
11226 .addImm(0)
11227 .add(predOps(ARMCC::AL));
11228 } else if (IsThumb1) {
11229 // load + update AddrIn
11230 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11231 .addReg(AddrIn)
11232 .addImm(0)
11233 .add(predOps(ARMCC::AL));
11234 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut)
11235 .add(t1CondCodeOp())
11236 .addReg(AddrIn)
11237 .addImm(LdSize)
11238 .add(predOps(ARMCC::AL));
11239 } else if (IsThumb2) {
11240 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11241 .addReg(AddrOut, RegState::Define)
11242 .addReg(AddrIn)
11243 .addImm(LdSize)
11244 .add(predOps(ARMCC::AL));
11245 } else { // arm
11246 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
11247 .addReg(AddrOut, RegState::Define)
11248 .addReg(AddrIn)
11249 .addReg(0)
11250 .addImm(LdSize)
11251 .add(predOps(ARMCC::AL));
11252 }
11253 }
11254
11255 /// Emit a post-increment store operation with given size. The instructions
11256 /// will be added to BB at Pos.
emitPostSt(MachineBasicBlock * BB,MachineBasicBlock::iterator Pos,const TargetInstrInfo * TII,const DebugLoc & dl,unsigned StSize,unsigned Data,unsigned AddrIn,unsigned AddrOut,bool IsThumb1,bool IsThumb2)11257 static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11258 const TargetInstrInfo *TII, const DebugLoc &dl,
11259 unsigned StSize, unsigned Data, unsigned AddrIn,
11260 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11261 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
11262 assert(StOpc != 0 && "Should have a store opcode");
11263 if (StSize >= 8) {
11264 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
11265 .addReg(AddrIn)
11266 .addImm(0)
11267 .addReg(Data)
11268 .add(predOps(ARMCC::AL));
11269 } else if (IsThumb1) {
11270 // store + update AddrIn
11271 BuildMI(*BB, Pos, dl, TII->get(StOpc))
11272 .addReg(Data)
11273 .addReg(AddrIn)
11274 .addImm(0)
11275 .add(predOps(ARMCC::AL));
11276 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut)
11277 .add(t1CondCodeOp())
11278 .addReg(AddrIn)
11279 .addImm(StSize)
11280 .add(predOps(ARMCC::AL));
11281 } else if (IsThumb2) {
11282 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
11283 .addReg(Data)
11284 .addReg(AddrIn)
11285 .addImm(StSize)
11286 .add(predOps(ARMCC::AL));
11287 } else { // arm
11288 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
11289 .addReg(Data)
11290 .addReg(AddrIn)
11291 .addReg(0)
11292 .addImm(StSize)
11293 .add(predOps(ARMCC::AL));
11294 }
11295 }
11296
11297 MachineBasicBlock *
EmitStructByval(MachineInstr & MI,MachineBasicBlock * BB) const11298 ARMTargetLowering::EmitStructByval(MachineInstr &MI,
11299 MachineBasicBlock *BB) const {
11300 // This pseudo instruction has 3 operands: dst, src, size
11301 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
11302 // Otherwise, we will generate unrolled scalar copies.
11303 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11304 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11305 MachineFunction::iterator It = ++BB->getIterator();
11306
11307 Register dest = MI.getOperand(0).getReg();
11308 Register src = MI.getOperand(1).getReg();
11309 unsigned SizeVal = MI.getOperand(2).getImm();
11310 unsigned Alignment = MI.getOperand(3).getImm();
11311 DebugLoc dl = MI.getDebugLoc();
11312
11313 MachineFunction *MF = BB->getParent();
11314 MachineRegisterInfo &MRI = MF->getRegInfo();
11315 unsigned UnitSize = 0;
11316 const TargetRegisterClass *TRC = nullptr;
11317 const TargetRegisterClass *VecTRC = nullptr;
11318
11319 bool IsThumb1 = Subtarget->isThumb1Only();
11320 bool IsThumb2 = Subtarget->isThumb2();
11321 bool IsThumb = Subtarget->isThumb();
11322
11323 if (Alignment & 1) {
11324 UnitSize = 1;
11325 } else if (Alignment & 2) {
11326 UnitSize = 2;
11327 } else {
11328 // Check whether we can use NEON instructions.
11329 if (!MF->getFunction().hasFnAttribute(Attribute::NoImplicitFloat) &&
11330 Subtarget->hasNEON()) {
11331 if ((Alignment % 16 == 0) && SizeVal >= 16)
11332 UnitSize = 16;
11333 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11334 UnitSize = 8;
11335 }
11336 // Can't use NEON instructions.
11337 if (UnitSize == 0)
11338 UnitSize = 4;
11339 }
11340
11341 // Select the correct opcode and register class for unit size load/store
11342 bool IsNeon = UnitSize >= 8;
11343 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11344 if (IsNeon)
11345 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11346 : UnitSize == 8 ? &ARM::DPRRegClass
11347 : nullptr;
11348
11349 unsigned BytesLeft = SizeVal % UnitSize;
11350 unsigned LoopSize = SizeVal - BytesLeft;
11351
11352 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11353 // Use LDR and STR to copy.
11354 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
11355 // [destOut] = STR_POST(scratch, destIn, UnitSize)
11356 unsigned srcIn = src;
11357 unsigned destIn = dest;
11358 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
11359 Register srcOut = MRI.createVirtualRegister(TRC);
11360 Register destOut = MRI.createVirtualRegister(TRC);
11361 Register scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11362 emitPostLd(BB, MI, TII, dl, UnitSize, scratch, srcIn, srcOut,
11363 IsThumb1, IsThumb2);
11364 emitPostSt(BB, MI, TII, dl, UnitSize, scratch, destIn, destOut,
11365 IsThumb1, IsThumb2);
11366 srcIn = srcOut;
11367 destIn = destOut;
11368 }
11369
11370 // Handle the leftover bytes with LDRB and STRB.
11371 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
11372 // [destOut] = STRB_POST(scratch, destIn, 1)
11373 for (unsigned i = 0; i < BytesLeft; i++) {
11374 Register srcOut = MRI.createVirtualRegister(TRC);
11375 Register destOut = MRI.createVirtualRegister(TRC);
11376 Register scratch = MRI.createVirtualRegister(TRC);
11377 emitPostLd(BB, MI, TII, dl, 1, scratch, srcIn, srcOut,
11378 IsThumb1, IsThumb2);
11379 emitPostSt(BB, MI, TII, dl, 1, scratch, destIn, destOut,
11380 IsThumb1, IsThumb2);
11381 srcIn = srcOut;
11382 destIn = destOut;
11383 }
11384 MI.eraseFromParent(); // The instruction is gone now.
11385 return BB;
11386 }
11387
11388 // Expand the pseudo op to a loop.
11389 // thisMBB:
11390 // ...
11391 // movw varEnd, # --> with thumb2
11392 // movt varEnd, #
11393 // ldrcp varEnd, idx --> without thumb2
11394 // fallthrough --> loopMBB
11395 // loopMBB:
11396 // PHI varPhi, varEnd, varLoop
11397 // PHI srcPhi, src, srcLoop
11398 // PHI destPhi, dst, destLoop
11399 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11400 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
11401 // subs varLoop, varPhi, #UnitSize
11402 // bne loopMBB
11403 // fallthrough --> exitMBB
11404 // exitMBB:
11405 // epilogue to handle left-over bytes
11406 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11407 // [destOut] = STRB_POST(scratch, destLoop, 1)
11408 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
11409 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
11410 MF->insert(It, loopMBB);
11411 MF->insert(It, exitMBB);
11412
11413 // Transfer the remainder of BB and its successor edges to exitMBB.
11414 exitMBB->splice(exitMBB->begin(), BB,
11415 std::next(MachineBasicBlock::iterator(MI)), BB->end());
11416 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
11417
11418 // Load an immediate to varEnd.
11419 Register varEnd = MRI.createVirtualRegister(TRC);
11420 if (Subtarget->useMovt()) {
11421 unsigned Vtmp = varEnd;
11422 if ((LoopSize & 0xFFFF0000) != 0)
11423 Vtmp = MRI.createVirtualRegister(TRC);
11424 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVi16 : ARM::MOVi16), Vtmp)
11425 .addImm(LoopSize & 0xFFFF)
11426 .add(predOps(ARMCC::AL));
11427
11428 if ((LoopSize & 0xFFFF0000) != 0)
11429 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVTi16 : ARM::MOVTi16), varEnd)
11430 .addReg(Vtmp)
11431 .addImm(LoopSize >> 16)
11432 .add(predOps(ARMCC::AL));
11433 } else {
11434 MachineConstantPool *ConstantPool = MF->getConstantPool();
11435 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
11436 const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
11437
11438 // MachineConstantPool wants an explicit alignment.
11439 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11440 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11441 MachineMemOperand *CPMMO =
11442 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
11443 MachineMemOperand::MOLoad, 4, Align(4));
11444
11445 if (IsThumb)
11446 BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci))
11447 .addReg(varEnd, RegState::Define)
11448 .addConstantPoolIndex(Idx)
11449 .add(predOps(ARMCC::AL))
11450 .addMemOperand(CPMMO);
11451 else
11452 BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp))
11453 .addReg(varEnd, RegState::Define)
11454 .addConstantPoolIndex(Idx)
11455 .addImm(0)
11456 .add(predOps(ARMCC::AL))
11457 .addMemOperand(CPMMO);
11458 }
11459 BB->addSuccessor(loopMBB);
11460
11461 // Generate the loop body:
11462 // varPhi = PHI(varLoop, varEnd)
11463 // srcPhi = PHI(srcLoop, src)
11464 // destPhi = PHI(destLoop, dst)
11465 MachineBasicBlock *entryBB = BB;
11466 BB = loopMBB;
11467 Register varLoop = MRI.createVirtualRegister(TRC);
11468 Register varPhi = MRI.createVirtualRegister(TRC);
11469 Register srcLoop = MRI.createVirtualRegister(TRC);
11470 Register srcPhi = MRI.createVirtualRegister(TRC);
11471 Register destLoop = MRI.createVirtualRegister(TRC);
11472 Register destPhi = MRI.createVirtualRegister(TRC);
11473
11474 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi)
11475 .addReg(varLoop).addMBB(loopMBB)
11476 .addReg(varEnd).addMBB(entryBB);
11477 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi)
11478 .addReg(srcLoop).addMBB(loopMBB)
11479 .addReg(src).addMBB(entryBB);
11480 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi)
11481 .addReg(destLoop).addMBB(loopMBB)
11482 .addReg(dest).addMBB(entryBB);
11483
11484 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11485 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
11486 Register scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11487 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop,
11488 IsThumb1, IsThumb2);
11489 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop,
11490 IsThumb1, IsThumb2);
11491
11492 // Decrement loop variable by UnitSize.
11493 if (IsThumb1) {
11494 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop)
11495 .add(t1CondCodeOp())
11496 .addReg(varPhi)
11497 .addImm(UnitSize)
11498 .add(predOps(ARMCC::AL));
11499 } else {
11500 MachineInstrBuilder MIB =
11501 BuildMI(*BB, BB->end(), dl,
11502 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11503 MIB.addReg(varPhi)
11504 .addImm(UnitSize)
11505 .add(predOps(ARMCC::AL))
11506 .add(condCodeOp());
11507 MIB->getOperand(5).setReg(ARM::CPSR);
11508 MIB->getOperand(5).setIsDef(true);
11509 }
11510 BuildMI(*BB, BB->end(), dl,
11511 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11512 .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR);
11513
11514 // loopMBB can loop back to loopMBB or fall through to exitMBB.
11515 BB->addSuccessor(loopMBB);
11516 BB->addSuccessor(exitMBB);
11517
11518 // Add epilogue to handle BytesLeft.
11519 BB = exitMBB;
11520 auto StartOfExit = exitMBB->begin();
11521
11522 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11523 // [destOut] = STRB_POST(scratch, destLoop, 1)
11524 unsigned srcIn = srcLoop;
11525 unsigned destIn = destLoop;
11526 for (unsigned i = 0; i < BytesLeft; i++) {
11527 Register srcOut = MRI.createVirtualRegister(TRC);
11528 Register destOut = MRI.createVirtualRegister(TRC);
11529 Register scratch = MRI.createVirtualRegister(TRC);
11530 emitPostLd(BB, StartOfExit, TII, dl, 1, scratch, srcIn, srcOut,
11531 IsThumb1, IsThumb2);
11532 emitPostSt(BB, StartOfExit, TII, dl, 1, scratch, destIn, destOut,
11533 IsThumb1, IsThumb2);
11534 srcIn = srcOut;
11535 destIn = destOut;
11536 }
11537
11538 MI.eraseFromParent(); // The instruction is gone now.
11539 return BB;
11540 }
11541
11542 MachineBasicBlock *
EmitLowered__chkstk(MachineInstr & MI,MachineBasicBlock * MBB) const11543 ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI,
11544 MachineBasicBlock *MBB) const {
11545 const TargetMachine &TM = getTargetMachine();
11546 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
11547 DebugLoc DL = MI.getDebugLoc();
11548
11549 assert(Subtarget->isTargetWindows() &&
11550 "__chkstk is only supported on Windows");
11551 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode");
11552
11553 // __chkstk takes the number of words to allocate on the stack in R4, and
11554 // returns the stack adjustment in number of bytes in R4. This will not
11555 // clober any other registers (other than the obvious lr).
11556 //
11557 // Although, technically, IP should be considered a register which may be
11558 // clobbered, the call itself will not touch it. Windows on ARM is a pure
11559 // thumb-2 environment, so there is no interworking required. As a result, we
11560 // do not expect a veneer to be emitted by the linker, clobbering IP.
11561 //
11562 // Each module receives its own copy of __chkstk, so no import thunk is
11563 // required, again, ensuring that IP is not clobbered.
11564 //
11565 // Finally, although some linkers may theoretically provide a trampoline for
11566 // out of range calls (which is quite common due to a 32M range limitation of
11567 // branches for Thumb), we can generate the long-call version via
11568 // -mcmodel=large, alleviating the need for the trampoline which may clobber
11569 // IP.
11570
11571 switch (TM.getCodeModel()) {
11572 case CodeModel::Tiny:
11573 llvm_unreachable("Tiny code model not available on ARM.");
11574 case CodeModel::Small:
11575 case CodeModel::Medium:
11576 case CodeModel::Kernel:
11577 BuildMI(*MBB, MI, DL, TII.get(ARM::tBL))
11578 .add(predOps(ARMCC::AL))
11579 .addExternalSymbol("__chkstk")
11580 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
11581 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
11582 .addReg(ARM::R12,
11583 RegState::Implicit | RegState::Define | RegState::Dead)
11584 .addReg(ARM::CPSR,
11585 RegState::Implicit | RegState::Define | RegState::Dead);
11586 break;
11587 case CodeModel::Large: {
11588 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
11589 Register Reg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11590
11591 BuildMI(*MBB, MI, DL, TII.get(ARM::t2MOVi32imm), Reg)
11592 .addExternalSymbol("__chkstk");
11593 BuildMI(*MBB, MI, DL, TII.get(gettBLXrOpcode(*MBB->getParent())))
11594 .add(predOps(ARMCC::AL))
11595 .addReg(Reg, RegState::Kill)
11596 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
11597 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
11598 .addReg(ARM::R12,
11599 RegState::Implicit | RegState::Define | RegState::Dead)
11600 .addReg(ARM::CPSR,
11601 RegState::Implicit | RegState::Define | RegState::Dead);
11602 break;
11603 }
11604 }
11605
11606 BuildMI(*MBB, MI, DL, TII.get(ARM::t2SUBrr), ARM::SP)
11607 .addReg(ARM::SP, RegState::Kill)
11608 .addReg(ARM::R4, RegState::Kill)
11609 .setMIFlags(MachineInstr::FrameSetup)
11610 .add(predOps(ARMCC::AL))
11611 .add(condCodeOp());
11612
11613 MI.eraseFromParent();
11614 return MBB;
11615 }
11616
11617 MachineBasicBlock *
EmitLowered__dbzchk(MachineInstr & MI,MachineBasicBlock * MBB) const11618 ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI,
11619 MachineBasicBlock *MBB) const {
11620 DebugLoc DL = MI.getDebugLoc();
11621 MachineFunction *MF = MBB->getParent();
11622 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11623
11624 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
11625 MF->insert(++MBB->getIterator(), ContBB);
11626 ContBB->splice(ContBB->begin(), MBB,
11627 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
11628 ContBB->transferSuccessorsAndUpdatePHIs(MBB);
11629 MBB->addSuccessor(ContBB);
11630
11631 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
11632 BuildMI(TrapBB, DL, TII->get(ARM::t__brkdiv0));
11633 MF->push_back(TrapBB);
11634 MBB->addSuccessor(TrapBB);
11635
11636 BuildMI(*MBB, MI, DL, TII->get(ARM::tCMPi8))
11637 .addReg(MI.getOperand(0).getReg())
11638 .addImm(0)
11639 .add(predOps(ARMCC::AL));
11640 BuildMI(*MBB, MI, DL, TII->get(ARM::t2Bcc))
11641 .addMBB(TrapBB)
11642 .addImm(ARMCC::EQ)
11643 .addReg(ARM::CPSR);
11644
11645 MI.eraseFromParent();
11646 return ContBB;
11647 }
11648
11649 // The CPSR operand of SelectItr might be missing a kill marker
11650 // because there were multiple uses of CPSR, and ISel didn't know
11651 // which to mark. Figure out whether SelectItr should have had a
11652 // kill marker, and set it if it should. Returns the correct kill
11653 // marker value.
checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr,MachineBasicBlock * BB,const TargetRegisterInfo * TRI)11654 static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr,
11655 MachineBasicBlock* BB,
11656 const TargetRegisterInfo* TRI) {
11657 // Scan forward through BB for a use/def of CPSR.
11658 MachineBasicBlock::iterator miI(std::next(SelectItr));
11659 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
11660 const MachineInstr& mi = *miI;
11661 if (mi.readsRegister(ARM::CPSR))
11662 return false;
11663 if (mi.definesRegister(ARM::CPSR))
11664 break; // Should have kill-flag - update below.
11665 }
11666
11667 // If we hit the end of the block, check whether CPSR is live into a
11668 // successor.
11669 if (miI == BB->end()) {
11670 for (MachineBasicBlock *Succ : BB->successors())
11671 if (Succ->isLiveIn(ARM::CPSR))
11672 return false;
11673 }
11674
11675 // We found a def, or hit the end of the basic block and CPSR wasn't live
11676 // out. SelectMI should have a kill flag on CPSR.
11677 SelectItr->addRegisterKilled(ARM::CPSR, TRI);
11678 return true;
11679 }
11680
11681 /// Adds logic in loop entry MBB to calculate loop iteration count and adds
11682 /// t2WhileLoopSetup and t2WhileLoopStart to generate WLS loop
genTPEntry(MachineBasicBlock * TpEntry,MachineBasicBlock * TpLoopBody,MachineBasicBlock * TpExit,Register OpSizeReg,const TargetInstrInfo * TII,DebugLoc Dl,MachineRegisterInfo & MRI)11683 static Register genTPEntry(MachineBasicBlock *TpEntry,
11684 MachineBasicBlock *TpLoopBody,
11685 MachineBasicBlock *TpExit, Register OpSizeReg,
11686 const TargetInstrInfo *TII, DebugLoc Dl,
11687 MachineRegisterInfo &MRI) {
11688 // Calculates loop iteration count = ceil(n/16) = (n + 15) >> 4.
11689 Register AddDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11690 BuildMI(TpEntry, Dl, TII->get(ARM::t2ADDri), AddDestReg)
11691 .addUse(OpSizeReg)
11692 .addImm(15)
11693 .add(predOps(ARMCC::AL))
11694 .addReg(0);
11695
11696 Register LsrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11697 BuildMI(TpEntry, Dl, TII->get(ARM::t2LSRri), LsrDestReg)
11698 .addUse(AddDestReg, RegState::Kill)
11699 .addImm(4)
11700 .add(predOps(ARMCC::AL))
11701 .addReg(0);
11702
11703 Register TotalIterationsReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11704 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11705 .addUse(LsrDestReg, RegState::Kill);
11706
11707 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopStart))
11708 .addUse(TotalIterationsReg)
11709 .addMBB(TpExit);
11710
11711 BuildMI(TpEntry, Dl, TII->get(ARM::t2B))
11712 .addMBB(TpLoopBody)
11713 .add(predOps(ARMCC::AL));
11714
11715 return TotalIterationsReg;
11716 }
11717
11718 /// Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and
11719 /// t2DoLoopEnd. These are used by later passes to generate tail predicated
11720 /// loops.
genTPLoopBody(MachineBasicBlock * TpLoopBody,MachineBasicBlock * TpEntry,MachineBasicBlock * TpExit,const TargetInstrInfo * TII,DebugLoc Dl,MachineRegisterInfo & MRI,Register OpSrcReg,Register OpDestReg,Register ElementCountReg,Register TotalIterationsReg,bool IsMemcpy)11721 static void genTPLoopBody(MachineBasicBlock *TpLoopBody,
11722 MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit,
11723 const TargetInstrInfo *TII, DebugLoc Dl,
11724 MachineRegisterInfo &MRI, Register OpSrcReg,
11725 Register OpDestReg, Register ElementCountReg,
11726 Register TotalIterationsReg, bool IsMemcpy) {
11727 // First insert 4 PHI nodes for: Current pointer to Src (if memcpy), Dest
11728 // array, loop iteration counter, predication counter.
11729
11730 Register SrcPhiReg, CurrSrcReg;
11731 if (IsMemcpy) {
11732 // Current position in the src array
11733 SrcPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11734 CurrSrcReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11735 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), SrcPhiReg)
11736 .addUse(OpSrcReg)
11737 .addMBB(TpEntry)
11738 .addUse(CurrSrcReg)
11739 .addMBB(TpLoopBody);
11740 }
11741
11742 // Current position in the dest array
11743 Register DestPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11744 Register CurrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11745 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), DestPhiReg)
11746 .addUse(OpDestReg)
11747 .addMBB(TpEntry)
11748 .addUse(CurrDestReg)
11749 .addMBB(TpLoopBody);
11750
11751 // Current loop counter
11752 Register LoopCounterPhiReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11753 Register RemainingLoopIterationsReg =
11754 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11755 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), LoopCounterPhiReg)
11756 .addUse(TotalIterationsReg)
11757 .addMBB(TpEntry)
11758 .addUse(RemainingLoopIterationsReg)
11759 .addMBB(TpLoopBody);
11760
11761 // Predication counter
11762 Register PredCounterPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11763 Register RemainingElementsReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11764 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), PredCounterPhiReg)
11765 .addUse(ElementCountReg)
11766 .addMBB(TpEntry)
11767 .addUse(RemainingElementsReg)
11768 .addMBB(TpLoopBody);
11769
11770 // Pass predication counter to VCTP
11771 Register VccrReg = MRI.createVirtualRegister(&ARM::VCCRRegClass);
11772 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VCTP8), VccrReg)
11773 .addUse(PredCounterPhiReg)
11774 .addImm(ARMVCC::None)
11775 .addReg(0)
11776 .addReg(0);
11777
11778 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2SUBri), RemainingElementsReg)
11779 .addUse(PredCounterPhiReg)
11780 .addImm(16)
11781 .add(predOps(ARMCC::AL))
11782 .addReg(0);
11783
11784 // VLDRB (only if memcpy) and VSTRB instructions, predicated using VPR
11785 Register SrcValueReg;
11786 if (IsMemcpy) {
11787 SrcValueReg = MRI.createVirtualRegister(&ARM::MQPRRegClass);
11788 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VLDRBU8_post))
11789 .addDef(CurrSrcReg)
11790 .addDef(SrcValueReg)
11791 .addReg(SrcPhiReg)
11792 .addImm(16)
11793 .addImm(ARMVCC::Then)
11794 .addUse(VccrReg)
11795 .addReg(0);
11796 } else
11797 SrcValueReg = OpSrcReg;
11798
11799 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VSTRBU8_post))
11800 .addDef(CurrDestReg)
11801 .addUse(SrcValueReg)
11802 .addReg(DestPhiReg)
11803 .addImm(16)
11804 .addImm(ARMVCC::Then)
11805 .addUse(VccrReg)
11806 .addReg(0);
11807
11808 // Add the pseudoInstrs for decrementing the loop counter and marking the
11809 // end:t2DoLoopDec and t2DoLoopEnd
11810 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11811 .addUse(LoopCounterPhiReg)
11812 .addImm(1);
11813
11814 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopEnd))
11815 .addUse(RemainingLoopIterationsReg)
11816 .addMBB(TpLoopBody);
11817
11818 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2B))
11819 .addMBB(TpExit)
11820 .add(predOps(ARMCC::AL));
11821 }
11822
11823 MachineBasicBlock *
EmitInstrWithCustomInserter(MachineInstr & MI,MachineBasicBlock * BB) const11824 ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
11825 MachineBasicBlock *BB) const {
11826 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11827 DebugLoc dl = MI.getDebugLoc();
11828 bool isThumb2 = Subtarget->isThumb2();
11829 switch (MI.getOpcode()) {
11830 default: {
11831 MI.print(errs());
11832 llvm_unreachable("Unexpected instr type to insert");
11833 }
11834
11835 // Thumb1 post-indexed loads are really just single-register LDMs.
11836 case ARM::tLDR_postidx: {
11837 MachineOperand Def(MI.getOperand(1));
11838 BuildMI(*BB, MI, dl, TII->get(ARM::tLDMIA_UPD))
11839 .add(Def) // Rn_wb
11840 .add(MI.getOperand(2)) // Rn
11841 .add(MI.getOperand(3)) // PredImm
11842 .add(MI.getOperand(4)) // PredReg
11843 .add(MI.getOperand(0)) // Rt
11844 .cloneMemRefs(MI);
11845 MI.eraseFromParent();
11846 return BB;
11847 }
11848
11849 case ARM::MVE_MEMCPYLOOPINST:
11850 case ARM::MVE_MEMSETLOOPINST: {
11851
11852 // Transformation below expands MVE_MEMCPYLOOPINST/MVE_MEMSETLOOPINST Pseudo
11853 // into a Tail Predicated (TP) Loop. It adds the instructions to calculate
11854 // the iteration count =ceil(size_in_bytes/16)) in the TP entry block and
11855 // adds the relevant instructions in the TP loop Body for generation of a
11856 // WLSTP loop.
11857
11858 // Below is relevant portion of the CFG after the transformation.
11859 // The Machine Basic Blocks are shown along with branch conditions (in
11860 // brackets). Note that TP entry/exit MBBs depict the entry/exit of this
11861 // portion of the CFG and may not necessarily be the entry/exit of the
11862 // function.
11863
11864 // (Relevant) CFG after transformation:
11865 // TP entry MBB
11866 // |
11867 // |-----------------|
11868 // (n <= 0) (n > 0)
11869 // | |
11870 // | TP loop Body MBB<--|
11871 // | | |
11872 // \ |___________|
11873 // \ /
11874 // TP exit MBB
11875
11876 MachineFunction *MF = BB->getParent();
11877 MachineFunctionProperties &Properties = MF->getProperties();
11878 MachineRegisterInfo &MRI = MF->getRegInfo();
11879
11880 Register OpDestReg = MI.getOperand(0).getReg();
11881 Register OpSrcReg = MI.getOperand(1).getReg();
11882 Register OpSizeReg = MI.getOperand(2).getReg();
11883
11884 // Allocate the required MBBs and add to parent function.
11885 MachineBasicBlock *TpEntry = BB;
11886 MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock();
11887 MachineBasicBlock *TpExit;
11888
11889 MF->push_back(TpLoopBody);
11890
11891 // If any instructions are present in the current block after
11892 // MVE_MEMCPYLOOPINST or MVE_MEMSETLOOPINST, split the current block and
11893 // move the instructions into the newly created exit block. If there are no
11894 // instructions add an explicit branch to the FallThrough block and then
11895 // split.
11896 //
11897 // The split is required for two reasons:
11898 // 1) A terminator(t2WhileLoopStart) will be placed at that site.
11899 // 2) Since a TPLoopBody will be added later, any phis in successive blocks
11900 // need to be updated. splitAt() already handles this.
11901 TpExit = BB->splitAt(MI, false);
11902 if (TpExit == BB) {
11903 assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the "
11904 "block containing memcpy/memset Pseudo");
11905 TpExit = BB->getFallThrough();
11906 BuildMI(BB, dl, TII->get(ARM::t2B))
11907 .addMBB(TpExit)
11908 .add(predOps(ARMCC::AL));
11909 TpExit = BB->splitAt(MI, false);
11910 }
11911
11912 // Add logic for iteration count
11913 Register TotalIterationsReg =
11914 genTPEntry(TpEntry, TpLoopBody, TpExit, OpSizeReg, TII, dl, MRI);
11915
11916 // Add the vectorized (and predicated) loads/store instructions
11917 bool IsMemcpy = MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
11918 genTPLoopBody(TpLoopBody, TpEntry, TpExit, TII, dl, MRI, OpSrcReg,
11919 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
11920
11921 // Required to avoid conflict with the MachineVerifier during testing.
11922 Properties.reset(MachineFunctionProperties::Property::NoPHIs);
11923
11924 // Connect the blocks
11925 TpEntry->addSuccessor(TpLoopBody);
11926 TpLoopBody->addSuccessor(TpLoopBody);
11927 TpLoopBody->addSuccessor(TpExit);
11928
11929 // Reorder for a more natural layout
11930 TpLoopBody->moveAfter(TpEntry);
11931 TpExit->moveAfter(TpLoopBody);
11932
11933 // Finally, remove the memcpy Psuedo Instruction
11934 MI.eraseFromParent();
11935
11936 // Return the exit block as it may contain other instructions requiring a
11937 // custom inserter
11938 return TpExit;
11939 }
11940
11941 // The Thumb2 pre-indexed stores have the same MI operands, they just
11942 // define them differently in the .td files from the isel patterns, so
11943 // they need pseudos.
11944 case ARM::t2STR_preidx:
11945 MI.setDesc(TII->get(ARM::t2STR_PRE));
11946 return BB;
11947 case ARM::t2STRB_preidx:
11948 MI.setDesc(TII->get(ARM::t2STRB_PRE));
11949 return BB;
11950 case ARM::t2STRH_preidx:
11951 MI.setDesc(TII->get(ARM::t2STRH_PRE));
11952 return BB;
11953
11954 case ARM::STRi_preidx:
11955 case ARM::STRBi_preidx: {
11956 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
11957 : ARM::STRB_PRE_IMM;
11958 // Decode the offset.
11959 unsigned Offset = MI.getOperand(4).getImm();
11960 bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub;
11961 Offset = ARM_AM::getAM2Offset(Offset);
11962 if (isSub)
11963 Offset = -Offset;
11964
11965 MachineMemOperand *MMO = *MI.memoperands_begin();
11966 BuildMI(*BB, MI, dl, TII->get(NewOpc))
11967 .add(MI.getOperand(0)) // Rn_wb
11968 .add(MI.getOperand(1)) // Rt
11969 .add(MI.getOperand(2)) // Rn
11970 .addImm(Offset) // offset (skip GPR==zero_reg)
11971 .add(MI.getOperand(5)) // pred
11972 .add(MI.getOperand(6))
11973 .addMemOperand(MMO);
11974 MI.eraseFromParent();
11975 return BB;
11976 }
11977 case ARM::STRr_preidx:
11978 case ARM::STRBr_preidx:
11979 case ARM::STRH_preidx: {
11980 unsigned NewOpc;
11981 switch (MI.getOpcode()) {
11982 default: llvm_unreachable("unexpected opcode!");
11983 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
11984 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
11985 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
11986 }
11987 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc));
11988 for (const MachineOperand &MO : MI.operands())
11989 MIB.add(MO);
11990 MI.eraseFromParent();
11991 return BB;
11992 }
11993
11994 case ARM::tMOVCCr_pseudo: {
11995 // To "insert" a SELECT_CC instruction, we actually have to insert the
11996 // diamond control-flow pattern. The incoming instruction knows the
11997 // destination vreg to set, the condition code register to branch on, the
11998 // true/false values to select between, and a branch opcode to use.
11999 const BasicBlock *LLVM_BB = BB->getBasicBlock();
12000 MachineFunction::iterator It = ++BB->getIterator();
12001
12002 // thisMBB:
12003 // ...
12004 // TrueVal = ...
12005 // cmpTY ccX, r1, r2
12006 // bCC copy1MBB
12007 // fallthrough --> copy0MBB
12008 MachineBasicBlock *thisMBB = BB;
12009 MachineFunction *F = BB->getParent();
12010 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
12011 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
12012 F->insert(It, copy0MBB);
12013 F->insert(It, sinkMBB);
12014
12015 // Check whether CPSR is live past the tMOVCCr_pseudo.
12016 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
12017 if (!MI.killsRegister(ARM::CPSR) &&
12018 !checkAndUpdateCPSRKill(MI, thisMBB, TRI)) {
12019 copy0MBB->addLiveIn(ARM::CPSR);
12020 sinkMBB->addLiveIn(ARM::CPSR);
12021 }
12022
12023 // Transfer the remainder of BB and its successor edges to sinkMBB.
12024 sinkMBB->splice(sinkMBB->begin(), BB,
12025 std::next(MachineBasicBlock::iterator(MI)), BB->end());
12026 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
12027
12028 BB->addSuccessor(copy0MBB);
12029 BB->addSuccessor(sinkMBB);
12030
12031 BuildMI(BB, dl, TII->get(ARM::tBcc))
12032 .addMBB(sinkMBB)
12033 .addImm(MI.getOperand(3).getImm())
12034 .addReg(MI.getOperand(4).getReg());
12035
12036 // copy0MBB:
12037 // %FalseValue = ...
12038 // # fallthrough to sinkMBB
12039 BB = copy0MBB;
12040
12041 // Update machine-CFG edges
12042 BB->addSuccessor(sinkMBB);
12043
12044 // sinkMBB:
12045 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
12046 // ...
12047 BB = sinkMBB;
12048 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg())
12049 .addReg(MI.getOperand(1).getReg())
12050 .addMBB(copy0MBB)
12051 .addReg(MI.getOperand(2).getReg())
12052 .addMBB(thisMBB);
12053
12054 MI.eraseFromParent(); // The pseudo instruction is gone now.
12055 return BB;
12056 }
12057
12058 case ARM::BCCi64:
12059 case ARM::BCCZi64: {
12060 // If there is an unconditional branch to the other successor, remove it.
12061 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end());
12062
12063 // Compare both parts that make up the double comparison separately for
12064 // equality.
12065 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64;
12066
12067 Register LHS1 = MI.getOperand(1).getReg();
12068 Register LHS2 = MI.getOperand(2).getReg();
12069 if (RHSisZero) {
12070 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12071 .addReg(LHS1)
12072 .addImm(0)
12073 .add(predOps(ARMCC::AL));
12074 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12075 .addReg(LHS2).addImm(0)
12076 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
12077 } else {
12078 Register RHS1 = MI.getOperand(3).getReg();
12079 Register RHS2 = MI.getOperand(4).getReg();
12080 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12081 .addReg(LHS1)
12082 .addReg(RHS1)
12083 .add(predOps(ARMCC::AL));
12084 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12085 .addReg(LHS2).addReg(RHS2)
12086 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
12087 }
12088
12089 MachineBasicBlock *destMBB = MI.getOperand(RHSisZero ? 3 : 5).getMBB();
12090 MachineBasicBlock *exitMBB = OtherSucc(BB, destMBB);
12091 if (MI.getOperand(0).getImm() == ARMCC::NE)
12092 std::swap(destMBB, exitMBB);
12093
12094 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12095 .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR);
12096 if (isThumb2)
12097 BuildMI(BB, dl, TII->get(ARM::t2B))
12098 .addMBB(exitMBB)
12099 .add(predOps(ARMCC::AL));
12100 else
12101 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB);
12102
12103 MI.eraseFromParent(); // The pseudo instruction is gone now.
12104 return BB;
12105 }
12106
12107 case ARM::Int_eh_sjlj_setjmp:
12108 case ARM::Int_eh_sjlj_setjmp_nofp:
12109 case ARM::tInt_eh_sjlj_setjmp:
12110 case ARM::t2Int_eh_sjlj_setjmp:
12111 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12112 return BB;
12113
12114 case ARM::Int_eh_sjlj_setup_dispatch:
12115 EmitSjLjDispatchBlock(MI, BB);
12116 return BB;
12117
12118 case ARM::ABS:
12119 case ARM::t2ABS: {
12120 // To insert an ABS instruction, we have to insert the
12121 // diamond control-flow pattern. The incoming instruction knows the
12122 // source vreg to test against 0, the destination vreg to set,
12123 // the condition code register to branch on, the
12124 // true/false values to select between, and a branch opcode to use.
12125 // It transforms
12126 // V1 = ABS V0
12127 // into
12128 // V2 = MOVS V0
12129 // BCC (branch to SinkBB if V0 >= 0)
12130 // RSBBB: V3 = RSBri V2, 0 (compute ABS if V2 < 0)
12131 // SinkBB: V1 = PHI(V2, V3)
12132 const BasicBlock *LLVM_BB = BB->getBasicBlock();
12133 MachineFunction::iterator BBI = ++BB->getIterator();
12134 MachineFunction *Fn = BB->getParent();
12135 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB);
12136 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB);
12137 Fn->insert(BBI, RSBBB);
12138 Fn->insert(BBI, SinkBB);
12139
12140 Register ABSSrcReg = MI.getOperand(1).getReg();
12141 Register ABSDstReg = MI.getOperand(0).getReg();
12142 bool ABSSrcKIll = MI.getOperand(1).isKill();
12143 bool isThumb2 = Subtarget->isThumb2();
12144 MachineRegisterInfo &MRI = Fn->getRegInfo();
12145 // In Thumb mode S must not be specified if source register is the SP or
12146 // PC and if destination register is the SP, so restrict register class
12147 Register NewRsbDstReg = MRI.createVirtualRegister(
12148 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12149
12150 // Transfer the remainder of BB and its successor edges to sinkMBB.
12151 SinkBB->splice(SinkBB->begin(), BB,
12152 std::next(MachineBasicBlock::iterator(MI)), BB->end());
12153 SinkBB->transferSuccessorsAndUpdatePHIs(BB);
12154
12155 BB->addSuccessor(RSBBB);
12156 BB->addSuccessor(SinkBB);
12157
12158 // fall through to SinkMBB
12159 RSBBB->addSuccessor(SinkBB);
12160
12161 // insert a cmp at the end of BB
12162 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12163 .addReg(ABSSrcReg)
12164 .addImm(0)
12165 .add(predOps(ARMCC::AL));
12166
12167 // insert a bcc with opposite CC to ARMCC::MI at the end of BB
12168 BuildMI(BB, dl,
12169 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB)
12170 .addImm(ARMCC::getOppositeCondition(ARMCC::MI)).addReg(ARM::CPSR);
12171
12172 // insert rsbri in RSBBB
12173 // Note: BCC and rsbri will be converted into predicated rsbmi
12174 // by if-conversion pass
12175 BuildMI(*RSBBB, RSBBB->begin(), dl,
12176 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12177 .addReg(ABSSrcReg, ABSSrcKIll ? RegState::Kill : 0)
12178 .addImm(0)
12179 .add(predOps(ARMCC::AL))
12180 .add(condCodeOp());
12181
12182 // insert PHI in SinkBB,
12183 // reuse ABSDstReg to not change uses of ABS instruction
12184 BuildMI(*SinkBB, SinkBB->begin(), dl,
12185 TII->get(ARM::PHI), ABSDstReg)
12186 .addReg(NewRsbDstReg).addMBB(RSBBB)
12187 .addReg(ABSSrcReg).addMBB(BB);
12188
12189 // remove ABS instruction
12190 MI.eraseFromParent();
12191
12192 // return last added BB
12193 return SinkBB;
12194 }
12195 case ARM::COPY_STRUCT_BYVAL_I32:
12196 ++NumLoopByVals;
12197 return EmitStructByval(MI, BB);
12198 case ARM::WIN__CHKSTK:
12199 return EmitLowered__chkstk(MI, BB);
12200 case ARM::WIN__DBZCHK:
12201 return EmitLowered__dbzchk(MI, BB);
12202 }
12203 }
12204
12205 /// Attaches vregs to MEMCPY that it will use as scratch registers
12206 /// when it is expanded into LDM/STM. This is done as a post-isel lowering
12207 /// instead of as a custom inserter because we need the use list from the SDNode.
attachMEMCPYScratchRegs(const ARMSubtarget * Subtarget,MachineInstr & MI,const SDNode * Node)12208 static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget,
12209 MachineInstr &MI, const SDNode *Node) {
12210 bool isThumb1 = Subtarget->isThumb1Only();
12211
12212 DebugLoc DL = MI.getDebugLoc();
12213 MachineFunction *MF = MI.getParent()->getParent();
12214 MachineRegisterInfo &MRI = MF->getRegInfo();
12215 MachineInstrBuilder MIB(*MF, MI);
12216
12217 // If the new dst/src is unused mark it as dead.
12218 if (!Node->hasAnyUseOfValue(0)) {
12219 MI.getOperand(0).setIsDead(true);
12220 }
12221 if (!Node->hasAnyUseOfValue(1)) {
12222 MI.getOperand(1).setIsDead(true);
12223 }
12224
12225 // The MEMCPY both defines and kills the scratch registers.
12226 for (unsigned I = 0; I != MI.getOperand(4).getImm(); ++I) {
12227 Register TmpReg = MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12228 : &ARM::GPRRegClass);
12229 MIB.addReg(TmpReg, RegState::Define|RegState::Dead);
12230 }
12231 }
12232
AdjustInstrPostInstrSelection(MachineInstr & MI,SDNode * Node) const12233 void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
12234 SDNode *Node) const {
12235 if (MI.getOpcode() == ARM::MEMCPY) {
12236 attachMEMCPYScratchRegs(Subtarget, MI, Node);
12237 return;
12238 }
12239
12240 const MCInstrDesc *MCID = &MI.getDesc();
12241 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
12242 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
12243 // operand is still set to noreg. If needed, set the optional operand's
12244 // register to CPSR, and remove the redundant implicit def.
12245 //
12246 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR).
12247
12248 // Rename pseudo opcodes.
12249 unsigned NewOpc = convertAddSubFlagsOpcode(MI.getOpcode());
12250 unsigned ccOutIdx;
12251 if (NewOpc) {
12252 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
12253 MCID = &TII->get(NewOpc);
12254
12255 assert(MCID->getNumOperands() ==
12256 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize()
12257 && "converted opcode should be the same except for cc_out"
12258 " (and, on Thumb1, pred)");
12259
12260 MI.setDesc(*MCID);
12261
12262 // Add the optional cc_out operand
12263 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/true));
12264
12265 // On Thumb1, move all input operands to the end, then add the predicate
12266 if (Subtarget->isThumb1Only()) {
12267 for (unsigned c = MCID->getNumOperands() - 4; c--;) {
12268 MI.addOperand(MI.getOperand(1));
12269 MI.removeOperand(1);
12270 }
12271
12272 // Restore the ties
12273 for (unsigned i = MI.getNumOperands(); i--;) {
12274 const MachineOperand& op = MI.getOperand(i);
12275 if (op.isReg() && op.isUse()) {
12276 int DefIdx = MCID->getOperandConstraint(i, MCOI::TIED_TO);
12277 if (DefIdx != -1)
12278 MI.tieOperands(DefIdx, i);
12279 }
12280 }
12281
12282 MI.addOperand(MachineOperand::CreateImm(ARMCC::AL));
12283 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/false));
12284 ccOutIdx = 1;
12285 } else
12286 ccOutIdx = MCID->getNumOperands() - 1;
12287 } else
12288 ccOutIdx = MCID->getNumOperands() - 1;
12289
12290 // Any ARM instruction that sets the 's' bit should specify an optional
12291 // "cc_out" operand in the last operand position.
12292 if (!MI.hasOptionalDef() || !MCID->operands()[ccOutIdx].isOptionalDef()) {
12293 assert(!NewOpc && "Optional cc_out operand required");
12294 return;
12295 }
12296 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
12297 // since we already have an optional CPSR def.
12298 bool definesCPSR = false;
12299 bool deadCPSR = false;
12300 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e;
12301 ++i) {
12302 const MachineOperand &MO = MI.getOperand(i);
12303 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
12304 definesCPSR = true;
12305 if (MO.isDead())
12306 deadCPSR = true;
12307 MI.removeOperand(i);
12308 break;
12309 }
12310 }
12311 if (!definesCPSR) {
12312 assert(!NewOpc && "Optional cc_out operand required");
12313 return;
12314 }
12315 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
12316 if (deadCPSR) {
12317 assert(!MI.getOperand(ccOutIdx).getReg() &&
12318 "expect uninitialized optional cc_out operand");
12319 // Thumb1 instructions must have the S bit even if the CPSR is dead.
12320 if (!Subtarget->isThumb1Only())
12321 return;
12322 }
12323
12324 // If this instruction was defined with an optional CPSR def and its dag node
12325 // had a live implicit CPSR def, then activate the optional CPSR def.
12326 MachineOperand &MO = MI.getOperand(ccOutIdx);
12327 MO.setReg(ARM::CPSR);
12328 MO.setIsDef(true);
12329 }
12330
12331 //===----------------------------------------------------------------------===//
12332 // ARM Optimization Hooks
12333 //===----------------------------------------------------------------------===//
12334
12335 // Helper function that checks if N is a null or all ones constant.
isZeroOrAllOnes(SDValue N,bool AllOnes)12336 static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
12337 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
12338 }
12339
12340 // Return true if N is conditionally 0 or all ones.
12341 // Detects these expressions where cc is an i1 value:
12342 //
12343 // (select cc 0, y) [AllOnes=0]
12344 // (select cc y, 0) [AllOnes=0]
12345 // (zext cc) [AllOnes=0]
12346 // (sext cc) [AllOnes=0/1]
12347 // (select cc -1, y) [AllOnes=1]
12348 // (select cc y, -1) [AllOnes=1]
12349 //
12350 // Invert is set when N is the null/all ones constant when CC is false.
12351 // OtherOp is set to the alternative value of N.
isConditionalZeroOrAllOnes(SDNode * N,bool AllOnes,SDValue & CC,bool & Invert,SDValue & OtherOp,SelectionDAG & DAG)12352 static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
12353 SDValue &CC, bool &Invert,
12354 SDValue &OtherOp,
12355 SelectionDAG &DAG) {
12356 switch (N->getOpcode()) {
12357 default: return false;
12358 case ISD::SELECT: {
12359 CC = N->getOperand(0);
12360 SDValue N1 = N->getOperand(1);
12361 SDValue N2 = N->getOperand(2);
12362 if (isZeroOrAllOnes(N1, AllOnes)) {
12363 Invert = false;
12364 OtherOp = N2;
12365 return true;
12366 }
12367 if (isZeroOrAllOnes(N2, AllOnes)) {
12368 Invert = true;
12369 OtherOp = N1;
12370 return true;
12371 }
12372 return false;
12373 }
12374 case ISD::ZERO_EXTEND:
12375 // (zext cc) can never be the all ones value.
12376 if (AllOnes)
12377 return false;
12378 [[fallthrough]];
12379 case ISD::SIGN_EXTEND: {
12380 SDLoc dl(N);
12381 EVT VT = N->getValueType(0);
12382 CC = N->getOperand(0);
12383 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC)
12384 return false;
12385 Invert = !AllOnes;
12386 if (AllOnes)
12387 // When looking for an AllOnes constant, N is an sext, and the 'other'
12388 // value is 0.
12389 OtherOp = DAG.getConstant(0, dl, VT);
12390 else if (N->getOpcode() == ISD::ZERO_EXTEND)
12391 // When looking for a 0 constant, N can be zext or sext.
12392 OtherOp = DAG.getConstant(1, dl, VT);
12393 else
12394 OtherOp = DAG.getAllOnesConstant(dl, VT);
12395 return true;
12396 }
12397 }
12398 }
12399
12400 // Combine a constant select operand into its use:
12401 //
12402 // (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
12403 // (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
12404 // (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
12405 // (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
12406 // (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
12407 //
12408 // The transform is rejected if the select doesn't have a constant operand that
12409 // is null, or all ones when AllOnes is set.
12410 //
12411 // Also recognize sext/zext from i1:
12412 //
12413 // (add (zext cc), x) -> (select cc (add x, 1), x)
12414 // (add (sext cc), x) -> (select cc (add x, -1), x)
12415 //
12416 // These transformations eventually create predicated instructions.
12417 //
12418 // @param N The node to transform.
12419 // @param Slct The N operand that is a select.
12420 // @param OtherOp The other N operand (x above).
12421 // @param DCI Context.
12422 // @param AllOnes Require the select constant to be all ones instead of null.
12423 // @returns The new node, or SDValue() on failure.
12424 static
combineSelectAndUse(SDNode * N,SDValue Slct,SDValue OtherOp,TargetLowering::DAGCombinerInfo & DCI,bool AllOnes=false)12425 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
12426 TargetLowering::DAGCombinerInfo &DCI,
12427 bool AllOnes = false) {
12428 SelectionDAG &DAG = DCI.DAG;
12429 EVT VT = N->getValueType(0);
12430 SDValue NonConstantVal;
12431 SDValue CCOp;
12432 bool SwapSelectOps;
12433 if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps,
12434 NonConstantVal, DAG))
12435 return SDValue();
12436
12437 // Slct is now know to be the desired identity constant when CC is true.
12438 SDValue TrueVal = OtherOp;
12439 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
12440 OtherOp, NonConstantVal);
12441 // Unless SwapSelectOps says CC should be false.
12442 if (SwapSelectOps)
12443 std::swap(TrueVal, FalseVal);
12444
12445 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
12446 CCOp, TrueVal, FalseVal);
12447 }
12448
12449 // Attempt combineSelectAndUse on each operand of a commutative operator N.
12450 static
combineSelectAndUseCommutative(SDNode * N,bool AllOnes,TargetLowering::DAGCombinerInfo & DCI)12451 SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
12452 TargetLowering::DAGCombinerInfo &DCI) {
12453 SDValue N0 = N->getOperand(0);
12454 SDValue N1 = N->getOperand(1);
12455 if (N0.getNode()->hasOneUse())
12456 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes))
12457 return Result;
12458 if (N1.getNode()->hasOneUse())
12459 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes))
12460 return Result;
12461 return SDValue();
12462 }
12463
IsVUZPShuffleNode(SDNode * N)12464 static bool IsVUZPShuffleNode(SDNode *N) {
12465 // VUZP shuffle node.
12466 if (N->getOpcode() == ARMISD::VUZP)
12467 return true;
12468
12469 // "VUZP" on i32 is an alias for VTRN.
12470 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(0) == MVT::v2i32)
12471 return true;
12472
12473 return false;
12474 }
12475
AddCombineToVPADD(SDNode * N,SDValue N0,SDValue N1,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)12476 static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
12477 TargetLowering::DAGCombinerInfo &DCI,
12478 const ARMSubtarget *Subtarget) {
12479 // Look for ADD(VUZP.0, VUZP.1).
12480 if (!IsVUZPShuffleNode(N0.getNode()) || N0.getNode() != N1.getNode() ||
12481 N0 == N1)
12482 return SDValue();
12483
12484 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD.
12485 if (!N->getValueType(0).is64BitVector())
12486 return SDValue();
12487
12488 // Generate vpadd.
12489 SelectionDAG &DAG = DCI.DAG;
12490 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12491 SDLoc dl(N);
12492 SDNode *Unzip = N0.getNode();
12493 EVT VT = N->getValueType(0);
12494
12495 SmallVector<SDValue, 8> Ops;
12496 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpadd, dl,
12497 TLI.getPointerTy(DAG.getDataLayout())));
12498 Ops.push_back(Unzip->getOperand(0));
12499 Ops.push_back(Unzip->getOperand(1));
12500
12501 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
12502 }
12503
AddCombineVUZPToVPADDL(SDNode * N,SDValue N0,SDValue N1,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)12504 static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12505 TargetLowering::DAGCombinerInfo &DCI,
12506 const ARMSubtarget *Subtarget) {
12507 // Check for two extended operands.
12508 if (!(N0.getOpcode() == ISD::SIGN_EXTEND &&
12509 N1.getOpcode() == ISD::SIGN_EXTEND) &&
12510 !(N0.getOpcode() == ISD::ZERO_EXTEND &&
12511 N1.getOpcode() == ISD::ZERO_EXTEND))
12512 return SDValue();
12513
12514 SDValue N00 = N0.getOperand(0);
12515 SDValue N10 = N1.getOperand(0);
12516
12517 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1))
12518 if (!IsVUZPShuffleNode(N00.getNode()) || N00.getNode() != N10.getNode() ||
12519 N00 == N10)
12520 return SDValue();
12521
12522 // We only recognize Q register paddl here; this can't be reached until
12523 // after type legalization.
12524 if (!N00.getValueType().is64BitVector() ||
12525 !N0.getValueType().is128BitVector())
12526 return SDValue();
12527
12528 // Generate vpaddl.
12529 SelectionDAG &DAG = DCI.DAG;
12530 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12531 SDLoc dl(N);
12532 EVT VT = N->getValueType(0);
12533
12534 SmallVector<SDValue, 8> Ops;
12535 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension.
12536 unsigned Opcode;
12537 if (N0.getOpcode() == ISD::SIGN_EXTEND)
12538 Opcode = Intrinsic::arm_neon_vpaddls;
12539 else
12540 Opcode = Intrinsic::arm_neon_vpaddlu;
12541 Ops.push_back(DAG.getConstant(Opcode, dl,
12542 TLI.getPointerTy(DAG.getDataLayout())));
12543 EVT ElemTy = N00.getValueType().getVectorElementType();
12544 unsigned NumElts = VT.getVectorNumElements();
12545 EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), ElemTy, NumElts * 2);
12546 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT,
12547 N00.getOperand(0), N00.getOperand(1));
12548 Ops.push_back(Concat);
12549
12550 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
12551 }
12552
12553 // FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in
12554 // an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is
12555 // much easier to match.
12556 static SDValue
AddCombineBUILD_VECTORToVPADDL(SDNode * N,SDValue N0,SDValue N1,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)12557 AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12558 TargetLowering::DAGCombinerInfo &DCI,
12559 const ARMSubtarget *Subtarget) {
12560 // Only perform optimization if after legalize, and if NEON is available. We
12561 // also expected both operands to be BUILD_VECTORs.
12562 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
12563 || N0.getOpcode() != ISD::BUILD_VECTOR
12564 || N1.getOpcode() != ISD::BUILD_VECTOR)
12565 return SDValue();
12566
12567 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
12568 EVT VT = N->getValueType(0);
12569 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
12570 return SDValue();
12571
12572 // Check that the vector operands are of the right form.
12573 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
12574 // operands, where N is the size of the formed vector.
12575 // Each EXTRACT_VECTOR should have the same input vector and odd or even
12576 // index such that we have a pair wise add pattern.
12577
12578 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
12579 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
12580 return SDValue();
12581 SDValue Vec = N0->getOperand(0)->getOperand(0);
12582 SDNode *V = Vec.getNode();
12583 unsigned nextIndex = 0;
12584
12585 // For each operands to the ADD which are BUILD_VECTORs,
12586 // check to see if each of their operands are an EXTRACT_VECTOR with
12587 // the same vector and appropriate index.
12588 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
12589 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT
12590 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
12591
12592 SDValue ExtVec0 = N0->getOperand(i);
12593 SDValue ExtVec1 = N1->getOperand(i);
12594
12595 // First operand is the vector, verify its the same.
12596 if (V != ExtVec0->getOperand(0).getNode() ||
12597 V != ExtVec1->getOperand(0).getNode())
12598 return SDValue();
12599
12600 // Second is the constant, verify its correct.
12601 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1));
12602 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1));
12603
12604 // For the constant, we want to see all the even or all the odd.
12605 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
12606 || C1->getZExtValue() != nextIndex+1)
12607 return SDValue();
12608
12609 // Increment index.
12610 nextIndex+=2;
12611 } else
12612 return SDValue();
12613 }
12614
12615 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure
12616 // we're using the entire input vector, otherwise there's a size/legality
12617 // mismatch somewhere.
12618 if (nextIndex != Vec.getValueType().getVectorNumElements() ||
12619 Vec.getValueType().getVectorElementType() == VT.getVectorElementType())
12620 return SDValue();
12621
12622 // Create VPADDL node.
12623 SelectionDAG &DAG = DCI.DAG;
12624 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12625
12626 SDLoc dl(N);
12627
12628 // Build operand list.
12629 SmallVector<SDValue, 8> Ops;
12630 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl,
12631 TLI.getPointerTy(DAG.getDataLayout())));
12632
12633 // Input is the vector.
12634 Ops.push_back(Vec);
12635
12636 // Get widened type and narrowed type.
12637 MVT widenType;
12638 unsigned numElem = VT.getVectorNumElements();
12639
12640 EVT inputLaneType = Vec.getValueType().getVectorElementType();
12641 switch (inputLaneType.getSimpleVT().SimpleTy) {
12642 case MVT::i8: widenType = MVT::getVectorVT(MVT::i16, numElem); break;
12643 case MVT::i16: widenType = MVT::getVectorVT(MVT::i32, numElem); break;
12644 case MVT::i32: widenType = MVT::getVectorVT(MVT::i64, numElem); break;
12645 default:
12646 llvm_unreachable("Invalid vector element type for padd optimization.");
12647 }
12648
12649 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
12650 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
12651 return DAG.getNode(ExtOp, dl, VT, tmp);
12652 }
12653
findMUL_LOHI(SDValue V)12654 static SDValue findMUL_LOHI(SDValue V) {
12655 if (V->getOpcode() == ISD::UMUL_LOHI ||
12656 V->getOpcode() == ISD::SMUL_LOHI)
12657 return V;
12658 return SDValue();
12659 }
12660
AddCombineTo64BitSMLAL16(SDNode * AddcNode,SDNode * AddeNode,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)12661 static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
12662 TargetLowering::DAGCombinerInfo &DCI,
12663 const ARMSubtarget *Subtarget) {
12664 if (!Subtarget->hasBaseDSP())
12665 return SDValue();
12666
12667 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and
12668 // accumulates the product into a 64-bit value. The 16-bit values will
12669 // be sign extended somehow or SRA'd into 32-bit values
12670 // (addc (adde (mul 16bit, 16bit), lo), hi)
12671 SDValue Mul = AddcNode->getOperand(0);
12672 SDValue Lo = AddcNode->getOperand(1);
12673 if (Mul.getOpcode() != ISD::MUL) {
12674 Lo = AddcNode->getOperand(0);
12675 Mul = AddcNode->getOperand(1);
12676 if (Mul.getOpcode() != ISD::MUL)
12677 return SDValue();
12678 }
12679
12680 SDValue SRA = AddeNode->getOperand(0);
12681 SDValue Hi = AddeNode->getOperand(1);
12682 if (SRA.getOpcode() != ISD::SRA) {
12683 SRA = AddeNode->getOperand(1);
12684 Hi = AddeNode->getOperand(0);
12685 if (SRA.getOpcode() != ISD::SRA)
12686 return SDValue();
12687 }
12688 if (auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12689 if (Const->getZExtValue() != 31)
12690 return SDValue();
12691 } else
12692 return SDValue();
12693
12694 if (SRA.getOperand(0) != Mul)
12695 return SDValue();
12696
12697 SelectionDAG &DAG = DCI.DAG;
12698 SDLoc dl(AddcNode);
12699 unsigned Opcode = 0;
12700 SDValue Op0;
12701 SDValue Op1;
12702
12703 if (isS16(Mul.getOperand(0), DAG) && isS16(Mul.getOperand(1), DAG)) {
12704 Opcode = ARMISD::SMLALBB;
12705 Op0 = Mul.getOperand(0);
12706 Op1 = Mul.getOperand(1);
12707 } else if (isS16(Mul.getOperand(0), DAG) && isSRA16(Mul.getOperand(1))) {
12708 Opcode = ARMISD::SMLALBT;
12709 Op0 = Mul.getOperand(0);
12710 Op1 = Mul.getOperand(1).getOperand(0);
12711 } else if (isSRA16(Mul.getOperand(0)) && isS16(Mul.getOperand(1), DAG)) {
12712 Opcode = ARMISD::SMLALTB;
12713 Op0 = Mul.getOperand(0).getOperand(0);
12714 Op1 = Mul.getOperand(1);
12715 } else if (isSRA16(Mul.getOperand(0)) && isSRA16(Mul.getOperand(1))) {
12716 Opcode = ARMISD::SMLALTT;
12717 Op0 = Mul->getOperand(0).getOperand(0);
12718 Op1 = Mul->getOperand(1).getOperand(0);
12719 }
12720
12721 if (!Op0 || !Op1)
12722 return SDValue();
12723
12724 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
12725 Op0, Op1, Lo, Hi);
12726 // Replace the ADDs' nodes uses by the MLA node's values.
12727 SDValue HiMLALResult(SMLAL.getNode(), 1);
12728 SDValue LoMLALResult(SMLAL.getNode(), 0);
12729
12730 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
12731 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
12732
12733 // Return original node to notify the driver to stop replacing.
12734 SDValue resNode(AddcNode, 0);
12735 return resNode;
12736 }
12737
AddCombineTo64bitMLAL(SDNode * AddeSubeNode,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)12738 static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode,
12739 TargetLowering::DAGCombinerInfo &DCI,
12740 const ARMSubtarget *Subtarget) {
12741 // Look for multiply add opportunities.
12742 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
12743 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
12744 // a glue link from the first add to the second add.
12745 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
12746 // a S/UMLAL instruction.
12747 // UMUL_LOHI
12748 // / :lo \ :hi
12749 // V \ [no multiline comment]
12750 // loAdd -> ADDC |
12751 // \ :carry /
12752 // V V
12753 // ADDE <- hiAdd
12754 //
12755 // In the special case where only the higher part of a signed result is used
12756 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts
12757 // a constant with the exact value of 0x80000000, we recognize we are dealing
12758 // with a "rounded multiply and add" (or subtract) and transform it into
12759 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively.
12760
12761 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE ||
12762 AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
12763 "Expect an ADDE or SUBE");
12764
12765 assert(AddeSubeNode->getNumOperands() == 3 &&
12766 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 &&
12767 "ADDE node has the wrong inputs");
12768
12769 // Check that we are chained to the right ADDC or SUBC node.
12770 SDNode *AddcSubcNode = AddeSubeNode->getOperand(2).getNode();
12771 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12772 AddcSubcNode->getOpcode() != ARMISD::ADDC) ||
12773 (AddeSubeNode->getOpcode() == ARMISD::SUBE &&
12774 AddcSubcNode->getOpcode() != ARMISD::SUBC))
12775 return SDValue();
12776
12777 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0);
12778 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1);
12779
12780 // Check if the two operands are from the same mul_lohi node.
12781 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode())
12782 return SDValue();
12783
12784 assert(AddcSubcNode->getNumValues() == 2 &&
12785 AddcSubcNode->getValueType(0) == MVT::i32 &&
12786 "Expect ADDC with two result values. First: i32");
12787
12788 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it
12789 // maybe a SMLAL which multiplies two 16-bit values.
12790 if (AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12791 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI &&
12792 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI &&
12793 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI &&
12794 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI)
12795 return AddCombineTo64BitSMLAL16(AddcSubcNode, AddeSubeNode, DCI, Subtarget);
12796
12797 // Check for the triangle shape.
12798 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0);
12799 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1);
12800
12801 // Make sure that the ADDE/SUBE operands are not coming from the same node.
12802 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode())
12803 return SDValue();
12804
12805 // Find the MUL_LOHI node walking up ADDE/SUBE's operands.
12806 bool IsLeftOperandMUL = false;
12807 SDValue MULOp = findMUL_LOHI(AddeSubeOp0);
12808 if (MULOp == SDValue())
12809 MULOp = findMUL_LOHI(AddeSubeOp1);
12810 else
12811 IsLeftOperandMUL = true;
12812 if (MULOp == SDValue())
12813 return SDValue();
12814
12815 // Figure out the right opcode.
12816 unsigned Opc = MULOp->getOpcode();
12817 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
12818
12819 // Figure out the high and low input values to the MLAL node.
12820 SDValue *HiAddSub = nullptr;
12821 SDValue *LoMul = nullptr;
12822 SDValue *LowAddSub = nullptr;
12823
12824 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI.
12825 if ((AddeSubeOp0 != MULOp.getValue(1)) && (AddeSubeOp1 != MULOp.getValue(1)))
12826 return SDValue();
12827
12828 if (IsLeftOperandMUL)
12829 HiAddSub = &AddeSubeOp1;
12830 else
12831 HiAddSub = &AddeSubeOp0;
12832
12833 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node
12834 // whose low result is fed to the ADDC/SUBC we are checking.
12835
12836 if (AddcSubcOp0 == MULOp.getValue(0)) {
12837 LoMul = &AddcSubcOp0;
12838 LowAddSub = &AddcSubcOp1;
12839 }
12840 if (AddcSubcOp1 == MULOp.getValue(0)) {
12841 LoMul = &AddcSubcOp1;
12842 LowAddSub = &AddcSubcOp0;
12843 }
12844
12845 if (!LoMul)
12846 return SDValue();
12847
12848 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC
12849 // the replacement below will create a cycle.
12850 if (AddcSubcNode == HiAddSub->getNode() ||
12851 AddcSubcNode->isPredecessorOf(HiAddSub->getNode()))
12852 return SDValue();
12853
12854 // Create the merged node.
12855 SelectionDAG &DAG = DCI.DAG;
12856
12857 // Start building operand list.
12858 SmallVector<SDValue, 8> Ops;
12859 Ops.push_back(LoMul->getOperand(0));
12860 Ops.push_back(LoMul->getOperand(1));
12861
12862 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be
12863 // the case, we must be doing signed multiplication and only use the higher
12864 // part of the result of the MLAL, furthermore the LowAddSub must be a constant
12865 // addition or subtraction with the value of 0x800000.
12866 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() &&
12867 FinalOpc == ARMISD::SMLAL && !AddeSubeNode->hasAnyUseOfValue(1) &&
12868 LowAddSub->getNode()->getOpcode() == ISD::Constant &&
12869 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() ==
12870 0x80000000) {
12871 Ops.push_back(*HiAddSub);
12872 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) {
12873 FinalOpc = ARMISD::SMMLSR;
12874 } else {
12875 FinalOpc = ARMISD::SMMLAR;
12876 }
12877 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops);
12878 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), NewNode);
12879
12880 return SDValue(AddeSubeNode, 0);
12881 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC)
12882 // SMMLS is generated during instruction selection and the rest of this
12883 // function can not handle the case where AddcSubcNode is a SUBC.
12884 return SDValue();
12885
12886 // Finish building the operand list for {U/S}MLAL
12887 Ops.push_back(*LowAddSub);
12888 Ops.push_back(*HiAddSub);
12889
12890 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode),
12891 DAG.getVTList(MVT::i32, MVT::i32), Ops);
12892
12893 // Replace the ADDs' nodes uses by the MLA node's values.
12894 SDValue HiMLALResult(MLALNode.getNode(), 1);
12895 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), HiMLALResult);
12896
12897 SDValue LoMLALResult(MLALNode.getNode(), 0);
12898 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcSubcNode, 0), LoMLALResult);
12899
12900 // Return original node to notify the driver to stop replacing.
12901 return SDValue(AddeSubeNode, 0);
12902 }
12903
AddCombineTo64bitUMAAL(SDNode * AddeNode,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)12904 static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode,
12905 TargetLowering::DAGCombinerInfo &DCI,
12906 const ARMSubtarget *Subtarget) {
12907 // UMAAL is similar to UMLAL except that it adds two unsigned values.
12908 // While trying to combine for the other MLAL nodes, first search for the
12909 // chance to use UMAAL. Check if Addc uses a node which has already
12910 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde
12911 // as the addend, and it's handled in PerformUMLALCombine.
12912
12913 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12914 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget);
12915
12916 // Check that we have a glued ADDC node.
12917 SDNode* AddcNode = AddeNode->getOperand(2).getNode();
12918 if (AddcNode->getOpcode() != ARMISD::ADDC)
12919 return SDValue();
12920
12921 // Find the converted UMAAL or quit if it doesn't exist.
12922 SDNode *UmlalNode = nullptr;
12923 SDValue AddHi;
12924 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) {
12925 UmlalNode = AddcNode->getOperand(0).getNode();
12926 AddHi = AddcNode->getOperand(1);
12927 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) {
12928 UmlalNode = AddcNode->getOperand(1).getNode();
12929 AddHi = AddcNode->getOperand(0);
12930 } else {
12931 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget);
12932 }
12933
12934 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as
12935 // the ADDC as well as Zero.
12936 if (!isNullConstant(UmlalNode->getOperand(3)))
12937 return SDValue();
12938
12939 if ((isNullConstant(AddeNode->getOperand(0)) &&
12940 AddeNode->getOperand(1).getNode() == UmlalNode) ||
12941 (AddeNode->getOperand(0).getNode() == UmlalNode &&
12942 isNullConstant(AddeNode->getOperand(1)))) {
12943 SelectionDAG &DAG = DCI.DAG;
12944 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1),
12945 UmlalNode->getOperand(2), AddHi };
12946 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode),
12947 DAG.getVTList(MVT::i32, MVT::i32), Ops);
12948
12949 // Replace the ADDs' nodes uses by the UMAAL node's values.
12950 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1));
12951 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0));
12952
12953 // Return original node to notify the driver to stop replacing.
12954 return SDValue(AddeNode, 0);
12955 }
12956 return SDValue();
12957 }
12958
PerformUMLALCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)12959 static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
12960 const ARMSubtarget *Subtarget) {
12961 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12962 return SDValue();
12963
12964 // Check that we have a pair of ADDC and ADDE as operands.
12965 // Both addends of the ADDE must be zero.
12966 SDNode* AddcNode = N->getOperand(2).getNode();
12967 SDNode* AddeNode = N->getOperand(3).getNode();
12968 if ((AddcNode->getOpcode() == ARMISD::ADDC) &&
12969 (AddeNode->getOpcode() == ARMISD::ADDE) &&
12970 isNullConstant(AddeNode->getOperand(0)) &&
12971 isNullConstant(AddeNode->getOperand(1)) &&
12972 (AddeNode->getOperand(2).getNode() == AddcNode))
12973 return DAG.getNode(ARMISD::UMAAL, SDLoc(N),
12974 DAG.getVTList(MVT::i32, MVT::i32),
12975 {N->getOperand(0), N->getOperand(1),
12976 AddcNode->getOperand(0), AddcNode->getOperand(1)});
12977 else
12978 return SDValue();
12979 }
12980
PerformAddcSubcCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)12981 static SDValue PerformAddcSubcCombine(SDNode *N,
12982 TargetLowering::DAGCombinerInfo &DCI,
12983 const ARMSubtarget *Subtarget) {
12984 SelectionDAG &DAG(DCI.DAG);
12985
12986 if (N->getOpcode() == ARMISD::SUBC && N->hasAnyUseOfValue(1)) {
12987 // (SUBC (ADDE 0, 0, C), 1) -> C
12988 SDValue LHS = N->getOperand(0);
12989 SDValue RHS = N->getOperand(1);
12990 if (LHS->getOpcode() == ARMISD::ADDE &&
12991 isNullConstant(LHS->getOperand(0)) &&
12992 isNullConstant(LHS->getOperand(1)) && isOneConstant(RHS)) {
12993 return DCI.CombineTo(N, SDValue(N, 0), LHS->getOperand(2));
12994 }
12995 }
12996
12997 if (Subtarget->isThumb1Only()) {
12998 SDValue RHS = N->getOperand(1);
12999 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
13000 int32_t imm = C->getSExtValue();
13001 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13002 SDLoc DL(N);
13003 RHS = DAG.getConstant(-imm, DL, MVT::i32);
13004 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC
13005 : ARMISD::ADDC;
13006 return DAG.getNode(Opcode, DL, N->getVTList(), N->getOperand(0), RHS);
13007 }
13008 }
13009 }
13010
13011 return SDValue();
13012 }
13013
PerformAddeSubeCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13014 static SDValue PerformAddeSubeCombine(SDNode *N,
13015 TargetLowering::DAGCombinerInfo &DCI,
13016 const ARMSubtarget *Subtarget) {
13017 if (Subtarget->isThumb1Only()) {
13018 SelectionDAG &DAG = DCI.DAG;
13019 SDValue RHS = N->getOperand(1);
13020 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
13021 int64_t imm = C->getSExtValue();
13022 if (imm < 0) {
13023 SDLoc DL(N);
13024
13025 // The with-carry-in form matches bitwise not instead of the negation.
13026 // Effectively, the inverse interpretation of the carry flag already
13027 // accounts for part of the negation.
13028 RHS = DAG.getConstant(~imm, DL, MVT::i32);
13029
13030 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE
13031 : ARMISD::ADDE;
13032 return DAG.getNode(Opcode, DL, N->getVTList(),
13033 N->getOperand(0), RHS, N->getOperand(2));
13034 }
13035 }
13036 } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) {
13037 return AddCombineTo64bitMLAL(N, DCI, Subtarget);
13038 }
13039 return SDValue();
13040 }
13041
PerformSELECTCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13042 static SDValue PerformSELECTCombine(SDNode *N,
13043 TargetLowering::DAGCombinerInfo &DCI,
13044 const ARMSubtarget *Subtarget) {
13045 if (!Subtarget->hasMVEIntegerOps())
13046 return SDValue();
13047
13048 SDLoc dl(N);
13049 SDValue SetCC;
13050 SDValue LHS;
13051 SDValue RHS;
13052 ISD::CondCode CC;
13053 SDValue TrueVal;
13054 SDValue FalseVal;
13055
13056 if (N->getOpcode() == ISD::SELECT &&
13057 N->getOperand(0)->getOpcode() == ISD::SETCC) {
13058 SetCC = N->getOperand(0);
13059 LHS = SetCC->getOperand(0);
13060 RHS = SetCC->getOperand(1);
13061 CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
13062 TrueVal = N->getOperand(1);
13063 FalseVal = N->getOperand(2);
13064 } else if (N->getOpcode() == ISD::SELECT_CC) {
13065 LHS = N->getOperand(0);
13066 RHS = N->getOperand(1);
13067 CC = cast<CondCodeSDNode>(N->getOperand(4))->get();
13068 TrueVal = N->getOperand(2);
13069 FalseVal = N->getOperand(3);
13070 } else {
13071 return SDValue();
13072 }
13073
13074 unsigned int Opcode = 0;
13075 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN ||
13076 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) &&
13077 (CC == ISD::SETULT || CC == ISD::SETUGT)) {
13078 Opcode = ARMISD::VMINVu;
13079 if (CC == ISD::SETUGT)
13080 std::swap(TrueVal, FalseVal);
13081 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN ||
13082 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) &&
13083 (CC == ISD::SETLT || CC == ISD::SETGT)) {
13084 Opcode = ARMISD::VMINVs;
13085 if (CC == ISD::SETGT)
13086 std::swap(TrueVal, FalseVal);
13087 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX ||
13088 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) &&
13089 (CC == ISD::SETUGT || CC == ISD::SETULT)) {
13090 Opcode = ARMISD::VMAXVu;
13091 if (CC == ISD::SETULT)
13092 std::swap(TrueVal, FalseVal);
13093 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX ||
13094 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) &&
13095 (CC == ISD::SETGT || CC == ISD::SETLT)) {
13096 Opcode = ARMISD::VMAXVs;
13097 if (CC == ISD::SETLT)
13098 std::swap(TrueVal, FalseVal);
13099 } else
13100 return SDValue();
13101
13102 // Normalise to the right hand side being the vector reduction
13103 switch (TrueVal->getOpcode()) {
13104 case ISD::VECREDUCE_UMIN:
13105 case ISD::VECREDUCE_SMIN:
13106 case ISD::VECREDUCE_UMAX:
13107 case ISD::VECREDUCE_SMAX:
13108 std::swap(LHS, RHS);
13109 std::swap(TrueVal, FalseVal);
13110 break;
13111 }
13112
13113 EVT VectorType = FalseVal->getOperand(0).getValueType();
13114
13115 if (VectorType != MVT::v16i8 && VectorType != MVT::v8i16 &&
13116 VectorType != MVT::v4i32)
13117 return SDValue();
13118
13119 EVT VectorScalarType = VectorType.getVectorElementType();
13120
13121 // The values being selected must also be the ones being compared
13122 if (TrueVal != LHS || FalseVal != RHS)
13123 return SDValue();
13124
13125 EVT LeftType = LHS->getValueType(0);
13126 EVT RightType = RHS->getValueType(0);
13127
13128 // The types must match the reduced type too
13129 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13130 return SDValue();
13131
13132 // Legalise the scalar to an i32
13133 if (VectorScalarType != MVT::i32)
13134 LHS = DCI.DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
13135
13136 // Generate the reduction as an i32 for legalisation purposes
13137 auto Reduction =
13138 DCI.DAG.getNode(Opcode, dl, MVT::i32, LHS, RHS->getOperand(0));
13139
13140 // The result isn't actually an i32 so truncate it back to its original type
13141 if (VectorScalarType != MVT::i32)
13142 Reduction = DCI.DAG.getNode(ISD::TRUNCATE, dl, VectorScalarType, Reduction);
13143
13144 return Reduction;
13145 }
13146
13147 // A special combine for the vqdmulh family of instructions. This is one of the
13148 // potential set of patterns that could patch this instruction. The base pattern
13149 // you would expect to be min(max(ashr(mul(mul(sext(x), 2), sext(y)), 16))).
13150 // This matches the different min(max(ashr(mul(mul(sext(x), sext(y)), 2), 16))),
13151 // which llvm will have optimized to min(ashr(mul(sext(x), sext(y)), 15))) as
13152 // the max is unnecessary.
PerformVQDMULHCombine(SDNode * N,SelectionDAG & DAG)13153 static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) {
13154 EVT VT = N->getValueType(0);
13155 SDValue Shft;
13156 ConstantSDNode *Clamp;
13157
13158 if (!VT.isVector() || VT.getScalarSizeInBits() > 64)
13159 return SDValue();
13160
13161 if (N->getOpcode() == ISD::SMIN) {
13162 Shft = N->getOperand(0);
13163 Clamp = isConstOrConstSplat(N->getOperand(1));
13164 } else if (N->getOpcode() == ISD::VSELECT) {
13165 // Detect a SMIN, which for an i64 node will be a vselect/setcc, not a smin.
13166 SDValue Cmp = N->getOperand(0);
13167 if (Cmp.getOpcode() != ISD::SETCC ||
13168 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() != ISD::SETLT ||
13169 Cmp.getOperand(0) != N->getOperand(1) ||
13170 Cmp.getOperand(1) != N->getOperand(2))
13171 return SDValue();
13172 Shft = N->getOperand(1);
13173 Clamp = isConstOrConstSplat(N->getOperand(2));
13174 } else
13175 return SDValue();
13176
13177 if (!Clamp)
13178 return SDValue();
13179
13180 MVT ScalarType;
13181 int ShftAmt = 0;
13182 switch (Clamp->getSExtValue()) {
13183 case (1 << 7) - 1:
13184 ScalarType = MVT::i8;
13185 ShftAmt = 7;
13186 break;
13187 case (1 << 15) - 1:
13188 ScalarType = MVT::i16;
13189 ShftAmt = 15;
13190 break;
13191 case (1ULL << 31) - 1:
13192 ScalarType = MVT::i32;
13193 ShftAmt = 31;
13194 break;
13195 default:
13196 return SDValue();
13197 }
13198
13199 if (Shft.getOpcode() != ISD::SRA)
13200 return SDValue();
13201 ConstantSDNode *N1 = isConstOrConstSplat(Shft.getOperand(1));
13202 if (!N1 || N1->getSExtValue() != ShftAmt)
13203 return SDValue();
13204
13205 SDValue Mul = Shft.getOperand(0);
13206 if (Mul.getOpcode() != ISD::MUL)
13207 return SDValue();
13208
13209 SDValue Ext0 = Mul.getOperand(0);
13210 SDValue Ext1 = Mul.getOperand(1);
13211 if (Ext0.getOpcode() != ISD::SIGN_EXTEND ||
13212 Ext1.getOpcode() != ISD::SIGN_EXTEND)
13213 return SDValue();
13214 EVT VecVT = Ext0.getOperand(0).getValueType();
13215 if (!VecVT.isPow2VectorType() || VecVT.getVectorNumElements() == 1)
13216 return SDValue();
13217 if (Ext1.getOperand(0).getValueType() != VecVT ||
13218 VecVT.getScalarType() != ScalarType ||
13219 VT.getScalarSizeInBits() < ScalarType.getScalarSizeInBits() * 2)
13220 return SDValue();
13221
13222 SDLoc DL(Mul);
13223 unsigned LegalLanes = 128 / (ShftAmt + 1);
13224 EVT LegalVecVT = MVT::getVectorVT(ScalarType, LegalLanes);
13225 // For types smaller than legal vectors extend to be legal and only use needed
13226 // lanes.
13227 if (VecVT.getSizeInBits() < 128) {
13228 EVT ExtVecVT =
13229 MVT::getVectorVT(MVT::getIntegerVT(128 / VecVT.getVectorNumElements()),
13230 VecVT.getVectorNumElements());
13231 SDValue Inp0 =
13232 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext0.getOperand(0));
13233 SDValue Inp1 =
13234 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext1.getOperand(0));
13235 Inp0 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp0);
13236 Inp1 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp1);
13237 SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1);
13238 SDValue Trunc = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, ExtVecVT, VQDMULH);
13239 Trunc = DAG.getNode(ISD::TRUNCATE, DL, VecVT, Trunc);
13240 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Trunc);
13241 }
13242
13243 // For larger types, split into legal sized chunks.
13244 assert(VecVT.getSizeInBits() % 128 == 0 && "Expected a power2 type");
13245 unsigned NumParts = VecVT.getSizeInBits() / 128;
13246 SmallVector<SDValue> Parts;
13247 for (unsigned I = 0; I < NumParts; ++I) {
13248 SDValue Inp0 =
13249 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext0.getOperand(0),
13250 DAG.getVectorIdxConstant(I * LegalLanes, DL));
13251 SDValue Inp1 =
13252 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext1.getOperand(0),
13253 DAG.getVectorIdxConstant(I * LegalLanes, DL));
13254 SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1);
13255 Parts.push_back(VQDMULH);
13256 }
13257 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT,
13258 DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Parts));
13259 }
13260
PerformVSELECTCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13261 static SDValue PerformVSELECTCombine(SDNode *N,
13262 TargetLowering::DAGCombinerInfo &DCI,
13263 const ARMSubtarget *Subtarget) {
13264 if (!Subtarget->hasMVEIntegerOps())
13265 return SDValue();
13266
13267 if (SDValue V = PerformVQDMULHCombine(N, DCI.DAG))
13268 return V;
13269
13270 // Transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).
13271 //
13272 // We need to re-implement this optimization here as the implementation in the
13273 // Target-Independent DAGCombiner does not handle the kind of constant we make
13274 // (it calls isConstOrConstSplat with AllowTruncation set to false - and for
13275 // good reason, allowing truncation there would break other targets).
13276 //
13277 // Currently, this is only done for MVE, as it's the only target that benefits
13278 // from this transformation (e.g. VPNOT+VPSEL becomes a single VPSEL).
13279 if (N->getOperand(0).getOpcode() != ISD::XOR)
13280 return SDValue();
13281 SDValue XOR = N->getOperand(0);
13282
13283 // Check if the XOR's RHS is either a 1, or a BUILD_VECTOR of 1s.
13284 // It is important to check with truncation allowed as the BUILD_VECTORs we
13285 // generate in those situations will truncate their operands.
13286 ConstantSDNode *Const =
13287 isConstOrConstSplat(XOR->getOperand(1), /*AllowUndefs*/ false,
13288 /*AllowTruncation*/ true);
13289 if (!Const || !Const->isOne())
13290 return SDValue();
13291
13292 // Rewrite into vselect(cond, rhs, lhs).
13293 SDValue Cond = XOR->getOperand(0);
13294 SDValue LHS = N->getOperand(1);
13295 SDValue RHS = N->getOperand(2);
13296 EVT Type = N->getValueType(0);
13297 return DCI.DAG.getNode(ISD::VSELECT, SDLoc(N), Type, Cond, RHS, LHS);
13298 }
13299
13300 // Convert vsetcc([0,1,2,..], splat(n), ult) -> vctp n
PerformVSetCCToVCTPCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13301 static SDValue PerformVSetCCToVCTPCombine(SDNode *N,
13302 TargetLowering::DAGCombinerInfo &DCI,
13303 const ARMSubtarget *Subtarget) {
13304 SDValue Op0 = N->getOperand(0);
13305 SDValue Op1 = N->getOperand(1);
13306 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
13307 EVT VT = N->getValueType(0);
13308
13309 if (!Subtarget->hasMVEIntegerOps() ||
13310 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
13311 return SDValue();
13312
13313 if (CC == ISD::SETUGE) {
13314 std::swap(Op0, Op1);
13315 CC = ISD::SETULT;
13316 }
13317
13318 if (CC != ISD::SETULT || VT.getScalarSizeInBits() != 1 ||
13319 Op0.getOpcode() != ISD::BUILD_VECTOR)
13320 return SDValue();
13321
13322 // Check first operand is BuildVector of 0,1,2,...
13323 for (unsigned I = 0; I < VT.getVectorNumElements(); I++) {
13324 if (!Op0.getOperand(I).isUndef() &&
13325 !(isa<ConstantSDNode>(Op0.getOperand(I)) &&
13326 Op0.getConstantOperandVal(I) == I))
13327 return SDValue();
13328 }
13329
13330 // The second is a Splat of Op1S
13331 SDValue Op1S = DCI.DAG.getSplatValue(Op1);
13332 if (!Op1S)
13333 return SDValue();
13334
13335 unsigned Opc;
13336 switch (VT.getVectorNumElements()) {
13337 case 2:
13338 Opc = Intrinsic::arm_mve_vctp64;
13339 break;
13340 case 4:
13341 Opc = Intrinsic::arm_mve_vctp32;
13342 break;
13343 case 8:
13344 Opc = Intrinsic::arm_mve_vctp16;
13345 break;
13346 case 16:
13347 Opc = Intrinsic::arm_mve_vctp8;
13348 break;
13349 default:
13350 return SDValue();
13351 }
13352
13353 SDLoc DL(N);
13354 return DCI.DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
13355 DCI.DAG.getConstant(Opc, DL, MVT::i32),
13356 DCI.DAG.getZExtOrTrunc(Op1S, DL, MVT::i32));
13357 }
13358
PerformABSCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13359 static SDValue PerformABSCombine(SDNode *N,
13360 TargetLowering::DAGCombinerInfo &DCI,
13361 const ARMSubtarget *Subtarget) {
13362 SelectionDAG &DAG = DCI.DAG;
13363 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13364
13365 if (TLI.isOperationLegal(N->getOpcode(), N->getValueType(0)))
13366 return SDValue();
13367
13368 return TLI.expandABS(N, DAG);
13369 }
13370
13371 /// PerformADDECombine - Target-specific dag combine transform from
13372 /// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
13373 /// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
PerformADDECombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13374 static SDValue PerformADDECombine(SDNode *N,
13375 TargetLowering::DAGCombinerInfo &DCI,
13376 const ARMSubtarget *Subtarget) {
13377 // Only ARM and Thumb2 support UMLAL/SMLAL.
13378 if (Subtarget->isThumb1Only())
13379 return PerformAddeSubeCombine(N, DCI, Subtarget);
13380
13381 // Only perform the checks after legalize when the pattern is available.
13382 if (DCI.isBeforeLegalize()) return SDValue();
13383
13384 return AddCombineTo64bitUMAAL(N, DCI, Subtarget);
13385 }
13386
13387 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
13388 /// operands N0 and N1. This is a helper for PerformADDCombine that is
13389 /// called with the default operands, and if that fails, with commuted
13390 /// operands.
PerformADDCombineWithOperands(SDNode * N,SDValue N0,SDValue N1,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13391 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
13392 TargetLowering::DAGCombinerInfo &DCI,
13393 const ARMSubtarget *Subtarget){
13394 // Attempt to create vpadd for this add.
13395 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
13396 return Result;
13397
13398 // Attempt to create vpaddl for this add.
13399 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
13400 return Result;
13401 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
13402 Subtarget))
13403 return Result;
13404
13405 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
13406 if (N0.getNode()->hasOneUse())
13407 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
13408 return Result;
13409 return SDValue();
13410 }
13411
TryDistrubutionADDVecReduce(SDNode * N,SelectionDAG & DAG)13412 static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG) {
13413 EVT VT = N->getValueType(0);
13414 SDValue N0 = N->getOperand(0);
13415 SDValue N1 = N->getOperand(1);
13416 SDLoc dl(N);
13417
13418 auto IsVecReduce = [](SDValue Op) {
13419 switch (Op.getOpcode()) {
13420 case ISD::VECREDUCE_ADD:
13421 case ARMISD::VADDVs:
13422 case ARMISD::VADDVu:
13423 case ARMISD::VMLAVs:
13424 case ARMISD::VMLAVu:
13425 return true;
13426 }
13427 return false;
13428 };
13429
13430 auto DistrubuteAddAddVecReduce = [&](SDValue N0, SDValue N1) {
13431 // Distribute add(X, add(vecreduce(Y), vecreduce(Z))) ->
13432 // add(add(X, vecreduce(Y)), vecreduce(Z))
13433 // to make better use of vaddva style instructions.
13434 if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) &&
13435 IsVecReduce(N1.getOperand(0)) && IsVecReduce(N1.getOperand(1)) &&
13436 !isa<ConstantSDNode>(N0) && N1->hasOneUse()) {
13437 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0, N1.getOperand(0));
13438 return DAG.getNode(ISD::ADD, dl, VT, Add0, N1.getOperand(1));
13439 }
13440 // And turn add(add(A, reduce(B)), add(C, reduce(D))) ->
13441 // add(add(add(A, C), reduce(B)), reduce(D))
13442 if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
13443 N1.getOpcode() == ISD::ADD && N0->hasOneUse() && N1->hasOneUse()) {
13444 unsigned N0RedOp = 0;
13445 if (!IsVecReduce(N0.getOperand(N0RedOp))) {
13446 N0RedOp = 1;
13447 if (!IsVecReduce(N0.getOperand(N0RedOp)))
13448 return SDValue();
13449 }
13450
13451 unsigned N1RedOp = 0;
13452 if (!IsVecReduce(N1.getOperand(N1RedOp)))
13453 N1RedOp = 1;
13454 if (!IsVecReduce(N1.getOperand(N1RedOp)))
13455 return SDValue();
13456
13457 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0.getOperand(1 - N0RedOp),
13458 N1.getOperand(1 - N1RedOp));
13459 SDValue Add1 =
13460 DAG.getNode(ISD::ADD, dl, VT, Add0, N0.getOperand(N0RedOp));
13461 return DAG.getNode(ISD::ADD, dl, VT, Add1, N1.getOperand(N1RedOp));
13462 }
13463 return SDValue();
13464 };
13465 if (SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13466 return R;
13467 if (SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13468 return R;
13469
13470 // Distribute add(vecreduce(load(Y)), vecreduce(load(Z)))
13471 // Or add(add(X, vecreduce(load(Y))), vecreduce(load(Z)))
13472 // by ascending load offsets. This can help cores prefetch if the order of
13473 // loads is more predictable.
13474 auto DistrubuteVecReduceLoad = [&](SDValue N0, SDValue N1, bool IsForward) {
13475 // Check if two reductions are known to load data where one is before/after
13476 // another. Return negative if N0 loads data before N1, positive if N1 is
13477 // before N0 and 0 otherwise if nothing is known.
13478 auto IsKnownOrderedLoad = [&](SDValue N0, SDValue N1) {
13479 // Look through to the first operand of a MUL, for the VMLA case.
13480 // Currently only looks at the first operand, in the hope they are equal.
13481 if (N0.getOpcode() == ISD::MUL)
13482 N0 = N0.getOperand(0);
13483 if (N1.getOpcode() == ISD::MUL)
13484 N1 = N1.getOperand(0);
13485
13486 // Return true if the two operands are loads to the same object and the
13487 // offset of the first is known to be less than the offset of the second.
13488 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13489 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13490 if (!Load0 || !Load1 || Load0->getChain() != Load1->getChain() ||
13491 !Load0->isSimple() || !Load1->isSimple() || Load0->isIndexed() ||
13492 Load1->isIndexed())
13493 return 0;
13494
13495 auto BaseLocDecomp0 = BaseIndexOffset::match(Load0, DAG);
13496 auto BaseLocDecomp1 = BaseIndexOffset::match(Load1, DAG);
13497
13498 if (!BaseLocDecomp0.getBase() ||
13499 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13500 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13501 return 0;
13502 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13503 return -1;
13504 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13505 return 1;
13506 return 0;
13507 };
13508
13509 SDValue X;
13510 if (N0.getOpcode() == ISD::ADD && N0->hasOneUse()) {
13511 if (IsVecReduce(N0.getOperand(0)) && IsVecReduce(N0.getOperand(1))) {
13512 int IsBefore = IsKnownOrderedLoad(N0.getOperand(0).getOperand(0),
13513 N0.getOperand(1).getOperand(0));
13514 if (IsBefore < 0) {
13515 X = N0.getOperand(0);
13516 N0 = N0.getOperand(1);
13517 } else if (IsBefore > 0) {
13518 X = N0.getOperand(1);
13519 N0 = N0.getOperand(0);
13520 } else
13521 return SDValue();
13522 } else if (IsVecReduce(N0.getOperand(0))) {
13523 X = N0.getOperand(1);
13524 N0 = N0.getOperand(0);
13525 } else if (IsVecReduce(N0.getOperand(1))) {
13526 X = N0.getOperand(0);
13527 N0 = N0.getOperand(1);
13528 } else
13529 return SDValue();
13530 } else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13531 IsKnownOrderedLoad(N0.getOperand(0), N1.getOperand(0)) < 0) {
13532 // Note this is backward to how you would expect. We create
13533 // add(reduce(load + 16), reduce(load + 0)) so that the
13534 // add(reduce(load+16), X) is combined into VADDVA(X, load+16)), leaving
13535 // the X as VADDV(load + 0)
13536 return DAG.getNode(ISD::ADD, dl, VT, N1, N0);
13537 } else
13538 return SDValue();
13539
13540 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13541 return SDValue();
13542
13543 if (IsKnownOrderedLoad(N1.getOperand(0), N0.getOperand(0)) >= 0)
13544 return SDValue();
13545
13546 // Switch from add(add(X, N0), N1) to add(add(X, N1), N0)
13547 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, X, N1);
13548 return DAG.getNode(ISD::ADD, dl, VT, Add0, N0);
13549 };
13550 if (SDValue R = DistrubuteVecReduceLoad(N0, N1, true))
13551 return R;
13552 if (SDValue R = DistrubuteVecReduceLoad(N1, N0, false))
13553 return R;
13554 return SDValue();
13555 }
13556
PerformADDVecReduce(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)13557 static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
13558 const ARMSubtarget *Subtarget) {
13559 if (!Subtarget->hasMVEIntegerOps())
13560 return SDValue();
13561
13562 if (SDValue R = TryDistrubutionADDVecReduce(N, DAG))
13563 return R;
13564
13565 EVT VT = N->getValueType(0);
13566 SDValue N0 = N->getOperand(0);
13567 SDValue N1 = N->getOperand(1);
13568 SDLoc dl(N);
13569
13570 if (VT != MVT::i64)
13571 return SDValue();
13572
13573 // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
13574 // will look like:
13575 // t1: i32,i32 = ARMISD::VADDLVs x
13576 // t2: i64 = build_pair t1, t1:1
13577 // t3: i64 = add t2, y
13578 // Otherwise we try to push the add up above VADDLVAx, to potentially allow
13579 // the add to be simplified seperately.
13580 // We also need to check for sext / zext and commutitive adds.
13581 auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
13582 SDValue NB) {
13583 if (NB->getOpcode() != ISD::BUILD_PAIR)
13584 return SDValue();
13585 SDValue VecRed = NB->getOperand(0);
13586 if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) ||
13587 VecRed.getResNo() != 0 ||
13588 NB->getOperand(1) != SDValue(VecRed.getNode(), 1))
13589 return SDValue();
13590
13591 if (VecRed->getOpcode() == OpcodeA) {
13592 // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y)
13593 SDValue Inp = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
13594 VecRed.getOperand(0), VecRed.getOperand(1));
13595 NA = DAG.getNode(ISD::ADD, dl, MVT::i64, Inp, NA);
13596 }
13597
13598 SmallVector<SDValue, 4> Ops;
13599 Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
13600 DAG.getConstant(0, dl, MVT::i32)));
13601 Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
13602 DAG.getConstant(1, dl, MVT::i32)));
13603 unsigned S = VecRed->getOpcode() == OpcodeA ? 2 : 0;
13604 for (unsigned I = S, E = VecRed.getNumOperands(); I < E; I++)
13605 Ops.push_back(VecRed->getOperand(I));
13606 SDValue Red =
13607 DAG.getNode(OpcodeA, dl, DAG.getVTList({MVT::i32, MVT::i32}), Ops);
13608 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red,
13609 SDValue(Red.getNode(), 1));
13610 };
13611
13612 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
13613 return M;
13614 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
13615 return M;
13616 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
13617 return M;
13618 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
13619 return M;
13620 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
13621 return M;
13622 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
13623 return M;
13624 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
13625 return M;
13626 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
13627 return M;
13628 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
13629 return M;
13630 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
13631 return M;
13632 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
13633 return M;
13634 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
13635 return M;
13636 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
13637 return M;
13638 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
13639 return M;
13640 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
13641 return M;
13642 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
13643 return M;
13644 return SDValue();
13645 }
13646
13647 bool
isDesirableToCommuteWithShift(const SDNode * N,CombineLevel Level) const13648 ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
13649 CombineLevel Level) const {
13650 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
13651 N->getOpcode() == ISD::SRL) &&
13652 "Expected shift op");
13653
13654 if (Level == BeforeLegalizeTypes)
13655 return true;
13656
13657 if (N->getOpcode() != ISD::SHL)
13658 return true;
13659
13660 if (Subtarget->isThumb1Only()) {
13661 // Avoid making expensive immediates by commuting shifts. (This logic
13662 // only applies to Thumb1 because ARM and Thumb2 immediates can be shifted
13663 // for free.)
13664 if (N->getOpcode() != ISD::SHL)
13665 return true;
13666 SDValue N1 = N->getOperand(0);
13667 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND &&
13668 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR)
13669 return true;
13670 if (auto *Const = dyn_cast<ConstantSDNode>(N1->getOperand(1))) {
13671 if (Const->getAPIntValue().ult(256))
13672 return false;
13673 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(0) &&
13674 Const->getAPIntValue().sgt(-256))
13675 return false;
13676 }
13677 return true;
13678 }
13679
13680 // Turn off commute-with-shift transform after legalization, so it doesn't
13681 // conflict with PerformSHLSimplify. (We could try to detect when
13682 // PerformSHLSimplify would trigger more precisely, but it isn't
13683 // really necessary.)
13684 return false;
13685 }
13686
isDesirableToCommuteXorWithShift(const SDNode * N) const13687 bool ARMTargetLowering::isDesirableToCommuteXorWithShift(
13688 const SDNode *N) const {
13689 assert(N->getOpcode() == ISD::XOR &&
13690 (N->getOperand(0).getOpcode() == ISD::SHL ||
13691 N->getOperand(0).getOpcode() == ISD::SRL) &&
13692 "Expected XOR(SHIFT) pattern");
13693
13694 // Only commute if the entire NOT mask is a hidden shifted mask.
13695 auto *XorC = dyn_cast<ConstantSDNode>(N->getOperand(1));
13696 auto *ShiftC = dyn_cast<ConstantSDNode>(N->getOperand(0).getOperand(1));
13697 if (XorC && ShiftC) {
13698 unsigned MaskIdx, MaskLen;
13699 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13700 unsigned ShiftAmt = ShiftC->getZExtValue();
13701 unsigned BitWidth = N->getValueType(0).getScalarSizeInBits();
13702 if (N->getOperand(0).getOpcode() == ISD::SHL)
13703 return MaskIdx == ShiftAmt && MaskLen == (BitWidth - ShiftAmt);
13704 return MaskIdx == 0 && MaskLen == (BitWidth - ShiftAmt);
13705 }
13706 }
13707
13708 return false;
13709 }
13710
shouldFoldConstantShiftPairToMask(const SDNode * N,CombineLevel Level) const13711 bool ARMTargetLowering::shouldFoldConstantShiftPairToMask(
13712 const SDNode *N, CombineLevel Level) const {
13713 assert(((N->getOpcode() == ISD::SHL &&
13714 N->getOperand(0).getOpcode() == ISD::SRL) ||
13715 (N->getOpcode() == ISD::SRL &&
13716 N->getOperand(0).getOpcode() == ISD::SHL)) &&
13717 "Expected shift-shift mask");
13718
13719 if (!Subtarget->isThumb1Only())
13720 return true;
13721
13722 if (Level == BeforeLegalizeTypes)
13723 return true;
13724
13725 return false;
13726 }
13727
preferIncOfAddToSubOfNot(EVT VT) const13728 bool ARMTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
13729 if (!Subtarget->hasNEON()) {
13730 if (Subtarget->isThumb1Only())
13731 return VT.getScalarSizeInBits() <= 32;
13732 return true;
13733 }
13734 return VT.isScalarInteger();
13735 }
13736
shouldConvertFpToSat(unsigned Op,EVT FPVT,EVT VT) const13737 bool ARMTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
13738 EVT VT) const {
13739 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
13740 return false;
13741
13742 switch (FPVT.getSimpleVT().SimpleTy) {
13743 case MVT::f16:
13744 return Subtarget->hasVFP2Base();
13745 case MVT::f32:
13746 return Subtarget->hasVFP2Base();
13747 case MVT::f64:
13748 return Subtarget->hasFP64();
13749 case MVT::v4f32:
13750 case MVT::v8f16:
13751 return Subtarget->hasMVEFloatOps();
13752 default:
13753 return false;
13754 }
13755 }
13756
PerformSHLSimplify(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * ST)13757 static SDValue PerformSHLSimplify(SDNode *N,
13758 TargetLowering::DAGCombinerInfo &DCI,
13759 const ARMSubtarget *ST) {
13760 // Allow the generic combiner to identify potential bswaps.
13761 if (DCI.isBeforeLegalize())
13762 return SDValue();
13763
13764 // DAG combiner will fold:
13765 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
13766 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2
13767 // Other code patterns that can be also be modified have the following form:
13768 // b + ((a << 1) | 510)
13769 // b + ((a << 1) & 510)
13770 // b + ((a << 1) ^ 510)
13771 // b + ((a << 1) + 510)
13772
13773 // Many instructions can perform the shift for free, but it requires both
13774 // the operands to be registers. If c1 << c2 is too large, a mov immediate
13775 // instruction will needed. So, unfold back to the original pattern if:
13776 // - if c1 and c2 are small enough that they don't require mov imms.
13777 // - the user(s) of the node can perform an shl
13778
13779 // No shifted operands for 16-bit instructions.
13780 if (ST->isThumb() && ST->isThumb1Only())
13781 return SDValue();
13782
13783 // Check that all the users could perform the shl themselves.
13784 for (auto *U : N->uses()) {
13785 switch(U->getOpcode()) {
13786 default:
13787 return SDValue();
13788 case ISD::SUB:
13789 case ISD::ADD:
13790 case ISD::AND:
13791 case ISD::OR:
13792 case ISD::XOR:
13793 case ISD::SETCC:
13794 case ARMISD::CMP:
13795 // Check that the user isn't already using a constant because there
13796 // aren't any instructions that support an immediate operand and a
13797 // shifted operand.
13798 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13799 isa<ConstantSDNode>(U->getOperand(1)))
13800 return SDValue();
13801
13802 // Check that it's not already using a shift.
13803 if (U->getOperand(0).getOpcode() == ISD::SHL ||
13804 U->getOperand(1).getOpcode() == ISD::SHL)
13805 return SDValue();
13806 break;
13807 }
13808 }
13809
13810 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR &&
13811 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND)
13812 return SDValue();
13813
13814 if (N->getOperand(0).getOpcode() != ISD::SHL)
13815 return SDValue();
13816
13817 SDValue SHL = N->getOperand(0);
13818
13819 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
13820 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13821 if (!C1ShlC2 || !C2)
13822 return SDValue();
13823
13824 APInt C2Int = C2->getAPIntValue();
13825 APInt C1Int = C1ShlC2->getAPIntValue();
13826 unsigned C2Width = C2Int.getBitWidth();
13827 if (C2Int.uge(C2Width))
13828 return SDValue();
13829 uint64_t C2Value = C2Int.getZExtValue();
13830
13831 // Check that performing a lshr will not lose any information.
13832 APInt Mask = APInt::getHighBitsSet(C2Width, C2Width - C2Value);
13833 if ((C1Int & Mask) != C1Int)
13834 return SDValue();
13835
13836 // Shift the first constant.
13837 C1Int.lshrInPlace(C2Int);
13838
13839 // The immediates are encoded as an 8-bit value that can be rotated.
13840 auto LargeImm = [](const APInt &Imm) {
13841 unsigned Zeros = Imm.countLeadingZeros() + Imm.countTrailingZeros();
13842 return Imm.getBitWidth() - Zeros > 8;
13843 };
13844
13845 if (LargeImm(C1Int) || LargeImm(C2Int))
13846 return SDValue();
13847
13848 SelectionDAG &DAG = DCI.DAG;
13849 SDLoc dl(N);
13850 SDValue X = SHL.getOperand(0);
13851 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X,
13852 DAG.getConstant(C1Int, dl, MVT::i32));
13853 // Shift left to compensate for the lshr of C1Int.
13854 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1));
13855
13856 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump();
13857 SHL.dump(); N->dump());
13858 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump());
13859 return Res;
13860 }
13861
13862
13863 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
13864 ///
PerformADDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13865 static SDValue PerformADDCombine(SDNode *N,
13866 TargetLowering::DAGCombinerInfo &DCI,
13867 const ARMSubtarget *Subtarget) {
13868 SDValue N0 = N->getOperand(0);
13869 SDValue N1 = N->getOperand(1);
13870
13871 // Only works one way, because it needs an immediate operand.
13872 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
13873 return Result;
13874
13875 if (SDValue Result = PerformADDVecReduce(N, DCI.DAG, Subtarget))
13876 return Result;
13877
13878 // First try with the default operand order.
13879 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
13880 return Result;
13881
13882 // If that didn't work, try again with the operands commuted.
13883 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget);
13884 }
13885
13886 // Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC)
13887 // providing -X is as cheap as X (currently, just a constant).
PerformSubCSINCCombine(SDNode * N,SelectionDAG & DAG)13888 static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG) {
13889 if (N->getValueType(0) != MVT::i32 || !isNullConstant(N->getOperand(0)))
13890 return SDValue();
13891 SDValue CSINC = N->getOperand(1);
13892 if (CSINC.getOpcode() != ARMISD::CSINC || !CSINC.hasOneUse())
13893 return SDValue();
13894
13895 ConstantSDNode *X = dyn_cast<ConstantSDNode>(CSINC.getOperand(0));
13896 if (!X)
13897 return SDValue();
13898
13899 return DAG.getNode(ARMISD::CSINV, SDLoc(N), MVT::i32,
13900 DAG.getNode(ISD::SUB, SDLoc(N), MVT::i32, N->getOperand(0),
13901 CSINC.getOperand(0)),
13902 CSINC.getOperand(1), CSINC.getOperand(2),
13903 CSINC.getOperand(3));
13904 }
13905
13906 /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
13907 ///
PerformSUBCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13908 static SDValue PerformSUBCombine(SDNode *N,
13909 TargetLowering::DAGCombinerInfo &DCI,
13910 const ARMSubtarget *Subtarget) {
13911 SDValue N0 = N->getOperand(0);
13912 SDValue N1 = N->getOperand(1);
13913
13914 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
13915 if (N1.getNode()->hasOneUse())
13916 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI))
13917 return Result;
13918
13919 if (SDValue R = PerformSubCSINCCombine(N, DCI.DAG))
13920 return R;
13921
13922 if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(0).isVector())
13923 return SDValue();
13924
13925 // Fold (sub (ARMvmovImm 0), (ARMvdup x)) -> (ARMvdup (sub 0, x))
13926 // so that we can readily pattern match more mve instructions which can use
13927 // a scalar operand.
13928 SDValue VDup = N->getOperand(1);
13929 if (VDup->getOpcode() != ARMISD::VDUP)
13930 return SDValue();
13931
13932 SDValue VMov = N->getOperand(0);
13933 if (VMov->getOpcode() == ISD::BITCAST)
13934 VMov = VMov->getOperand(0);
13935
13936 if (VMov->getOpcode() != ARMISD::VMOVIMM || !isZeroVector(VMov))
13937 return SDValue();
13938
13939 SDLoc dl(N);
13940 SDValue Negate = DCI.DAG.getNode(ISD::SUB, dl, MVT::i32,
13941 DCI.DAG.getConstant(0, dl, MVT::i32),
13942 VDup->getOperand(0));
13943 return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), Negate);
13944 }
13945
13946 /// PerformVMULCombine
13947 /// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
13948 /// special multiplier accumulator forwarding.
13949 /// vmul d3, d0, d2
13950 /// vmla d3, d1, d2
13951 /// is faster than
13952 /// vadd d3, d0, d1
13953 /// vmul d3, d3, d2
13954 // However, for (A + B) * (A + B),
13955 // vadd d2, d0, d1
13956 // vmul d3, d0, d2
13957 // vmla d3, d1, d2
13958 // is slower than
13959 // vadd d2, d0, d1
13960 // vmul d3, d2, d2
PerformVMULCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)13961 static SDValue PerformVMULCombine(SDNode *N,
13962 TargetLowering::DAGCombinerInfo &DCI,
13963 const ARMSubtarget *Subtarget) {
13964 if (!Subtarget->hasVMLxForwarding())
13965 return SDValue();
13966
13967 SelectionDAG &DAG = DCI.DAG;
13968 SDValue N0 = N->getOperand(0);
13969 SDValue N1 = N->getOperand(1);
13970 unsigned Opcode = N0.getOpcode();
13971 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
13972 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
13973 Opcode = N1.getOpcode();
13974 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
13975 Opcode != ISD::FADD && Opcode != ISD::FSUB)
13976 return SDValue();
13977 std::swap(N0, N1);
13978 }
13979
13980 if (N0 == N1)
13981 return SDValue();
13982
13983 EVT VT = N->getValueType(0);
13984 SDLoc DL(N);
13985 SDValue N00 = N0->getOperand(0);
13986 SDValue N01 = N0->getOperand(1);
13987 return DAG.getNode(Opcode, DL, VT,
13988 DAG.getNode(ISD::MUL, DL, VT, N00, N1),
13989 DAG.getNode(ISD::MUL, DL, VT, N01, N1));
13990 }
13991
PerformMVEVMULLCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)13992 static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
13993 const ARMSubtarget *Subtarget) {
13994 EVT VT = N->getValueType(0);
13995 if (VT != MVT::v2i64)
13996 return SDValue();
13997
13998 SDValue N0 = N->getOperand(0);
13999 SDValue N1 = N->getOperand(1);
14000
14001 auto IsSignExt = [&](SDValue Op) {
14002 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG)
14003 return SDValue();
14004 EVT VT = cast<VTSDNode>(Op->getOperand(1))->getVT();
14005 if (VT.getScalarSizeInBits() == 32)
14006 return Op->getOperand(0);
14007 return SDValue();
14008 };
14009 auto IsZeroExt = [&](SDValue Op) {
14010 // Zero extends are a little more awkward. At the point we are matching
14011 // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask.
14012 // That might be before of after a bitcast depending on how the and is
14013 // placed. Because this has to look through bitcasts, it is currently only
14014 // supported on LE.
14015 if (!Subtarget->isLittle())
14016 return SDValue();
14017
14018 SDValue And = Op;
14019 if (And->getOpcode() == ISD::BITCAST)
14020 And = And->getOperand(0);
14021 if (And->getOpcode() != ISD::AND)
14022 return SDValue();
14023 SDValue Mask = And->getOperand(1);
14024 if (Mask->getOpcode() == ISD::BITCAST)
14025 Mask = Mask->getOperand(0);
14026
14027 if (Mask->getOpcode() != ISD::BUILD_VECTOR ||
14028 Mask.getValueType() != MVT::v4i32)
14029 return SDValue();
14030 if (isAllOnesConstant(Mask->getOperand(0)) &&
14031 isNullConstant(Mask->getOperand(1)) &&
14032 isAllOnesConstant(Mask->getOperand(2)) &&
14033 isNullConstant(Mask->getOperand(3)))
14034 return And->getOperand(0);
14035 return SDValue();
14036 };
14037
14038 SDLoc dl(N);
14039 if (SDValue Op0 = IsSignExt(N0)) {
14040 if (SDValue Op1 = IsSignExt(N1)) {
14041 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14042 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14043 return DAG.getNode(ARMISD::VMULLs, dl, VT, New0a, New1a);
14044 }
14045 }
14046 if (SDValue Op0 = IsZeroExt(N0)) {
14047 if (SDValue Op1 = IsZeroExt(N1)) {
14048 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14049 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14050 return DAG.getNode(ARMISD::VMULLu, dl, VT, New0a, New1a);
14051 }
14052 }
14053
14054 return SDValue();
14055 }
14056
PerformMULCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14057 static SDValue PerformMULCombine(SDNode *N,
14058 TargetLowering::DAGCombinerInfo &DCI,
14059 const ARMSubtarget *Subtarget) {
14060 SelectionDAG &DAG = DCI.DAG;
14061
14062 EVT VT = N->getValueType(0);
14063 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14064 return PerformMVEVMULLCombine(N, DAG, Subtarget);
14065
14066 if (Subtarget->isThumb1Only())
14067 return SDValue();
14068
14069 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14070 return SDValue();
14071
14072 if (VT.is64BitVector() || VT.is128BitVector())
14073 return PerformVMULCombine(N, DCI, Subtarget);
14074 if (VT != MVT::i32)
14075 return SDValue();
14076
14077 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
14078 if (!C)
14079 return SDValue();
14080
14081 int64_t MulAmt = C->getSExtValue();
14082 unsigned ShiftAmt = countTrailingZeros<uint64_t>(MulAmt);
14083
14084 ShiftAmt = ShiftAmt & (32 - 1);
14085 SDValue V = N->getOperand(0);
14086 SDLoc DL(N);
14087
14088 SDValue Res;
14089 MulAmt >>= ShiftAmt;
14090
14091 if (MulAmt >= 0) {
14092 if (isPowerOf2_32(MulAmt - 1)) {
14093 // (mul x, 2^N + 1) => (add (shl x, N), x)
14094 Res = DAG.getNode(ISD::ADD, DL, VT,
14095 V,
14096 DAG.getNode(ISD::SHL, DL, VT,
14097 V,
14098 DAG.getConstant(Log2_32(MulAmt - 1), DL,
14099 MVT::i32)));
14100 } else if (isPowerOf2_32(MulAmt + 1)) {
14101 // (mul x, 2^N - 1) => (sub (shl x, N), x)
14102 Res = DAG.getNode(ISD::SUB, DL, VT,
14103 DAG.getNode(ISD::SHL, DL, VT,
14104 V,
14105 DAG.getConstant(Log2_32(MulAmt + 1), DL,
14106 MVT::i32)),
14107 V);
14108 } else
14109 return SDValue();
14110 } else {
14111 uint64_t MulAmtAbs = -MulAmt;
14112 if (isPowerOf2_32(MulAmtAbs + 1)) {
14113 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
14114 Res = DAG.getNode(ISD::SUB, DL, VT,
14115 V,
14116 DAG.getNode(ISD::SHL, DL, VT,
14117 V,
14118 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL,
14119 MVT::i32)));
14120 } else if (isPowerOf2_32(MulAmtAbs - 1)) {
14121 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
14122 Res = DAG.getNode(ISD::ADD, DL, VT,
14123 V,
14124 DAG.getNode(ISD::SHL, DL, VT,
14125 V,
14126 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL,
14127 MVT::i32)));
14128 Res = DAG.getNode(ISD::SUB, DL, VT,
14129 DAG.getConstant(0, DL, MVT::i32), Res);
14130 } else
14131 return SDValue();
14132 }
14133
14134 if (ShiftAmt != 0)
14135 Res = DAG.getNode(ISD::SHL, DL, VT,
14136 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32));
14137
14138 // Do not add new nodes to DAG combiner worklist.
14139 DCI.CombineTo(N, Res, false);
14140 return SDValue();
14141 }
14142
CombineANDShift(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14143 static SDValue CombineANDShift(SDNode *N,
14144 TargetLowering::DAGCombinerInfo &DCI,
14145 const ARMSubtarget *Subtarget) {
14146 // Allow DAGCombine to pattern-match before we touch the canonical form.
14147 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14148 return SDValue();
14149
14150 if (N->getValueType(0) != MVT::i32)
14151 return SDValue();
14152
14153 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
14154 if (!N1C)
14155 return SDValue();
14156
14157 uint32_t C1 = (uint32_t)N1C->getZExtValue();
14158 // Don't transform uxtb/uxth.
14159 if (C1 == 255 || C1 == 65535)
14160 return SDValue();
14161
14162 SDNode *N0 = N->getOperand(0).getNode();
14163 if (!N0->hasOneUse())
14164 return SDValue();
14165
14166 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL)
14167 return SDValue();
14168
14169 bool LeftShift = N0->getOpcode() == ISD::SHL;
14170
14171 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
14172 if (!N01C)
14173 return SDValue();
14174
14175 uint32_t C2 = (uint32_t)N01C->getZExtValue();
14176 if (!C2 || C2 >= 32)
14177 return SDValue();
14178
14179 // Clear irrelevant bits in the mask.
14180 if (LeftShift)
14181 C1 &= (-1U << C2);
14182 else
14183 C1 &= (-1U >> C2);
14184
14185 SelectionDAG &DAG = DCI.DAG;
14186 SDLoc DL(N);
14187
14188 // We have a pattern of the form "(and (shl x, c2) c1)" or
14189 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to
14190 // transform to a pair of shifts, to save materializing c1.
14191
14192 // First pattern: right shift, then mask off leading bits.
14193 // FIXME: Use demanded bits?
14194 if (!LeftShift && isMask_32(C1)) {
14195 uint32_t C3 = countLeadingZeros(C1);
14196 if (C2 < C3) {
14197 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
14198 DAG.getConstant(C3 - C2, DL, MVT::i32));
14199 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
14200 DAG.getConstant(C3, DL, MVT::i32));
14201 }
14202 }
14203
14204 // First pattern, reversed: left shift, then mask off trailing bits.
14205 if (LeftShift && isMask_32(~C1)) {
14206 uint32_t C3 = countTrailingZeros(C1);
14207 if (C2 < C3) {
14208 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
14209 DAG.getConstant(C3 - C2, DL, MVT::i32));
14210 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
14211 DAG.getConstant(C3, DL, MVT::i32));
14212 }
14213 }
14214
14215 // Second pattern: left shift, then mask off leading bits.
14216 // FIXME: Use demanded bits?
14217 if (LeftShift && isShiftedMask_32(C1)) {
14218 uint32_t Trailing = countTrailingZeros(C1);
14219 uint32_t C3 = countLeadingZeros(C1);
14220 if (Trailing == C2 && C2 + C3 < 32) {
14221 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
14222 DAG.getConstant(C2 + C3, DL, MVT::i32));
14223 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
14224 DAG.getConstant(C3, DL, MVT::i32));
14225 }
14226 }
14227
14228 // Second pattern, reversed: right shift, then mask off trailing bits.
14229 // FIXME: Handle other patterns of known/demanded bits.
14230 if (!LeftShift && isShiftedMask_32(C1)) {
14231 uint32_t Leading = countLeadingZeros(C1);
14232 uint32_t C3 = countTrailingZeros(C1);
14233 if (Leading == C2 && C2 + C3 < 32) {
14234 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
14235 DAG.getConstant(C2 + C3, DL, MVT::i32));
14236 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
14237 DAG.getConstant(C3, DL, MVT::i32));
14238 }
14239 }
14240
14241 // FIXME: Transform "(and (shl x, c2) c1)" ->
14242 // "(shl (and x, c1>>c2), c2)" if "c1 >> c2" is a cheaper immediate than
14243 // c1.
14244 return SDValue();
14245 }
14246
PerformANDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14247 static SDValue PerformANDCombine(SDNode *N,
14248 TargetLowering::DAGCombinerInfo &DCI,
14249 const ARMSubtarget *Subtarget) {
14250 // Attempt to use immediate-form VBIC
14251 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
14252 SDLoc dl(N);
14253 EVT VT = N->getValueType(0);
14254 SelectionDAG &DAG = DCI.DAG;
14255
14256 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 ||
14257 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14258 return SDValue();
14259
14260 APInt SplatBits, SplatUndef;
14261 unsigned SplatBitSize;
14262 bool HasAnyUndefs;
14263 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14264 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14265 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14266 SplatBitSize == 64) {
14267 EVT VbicVT;
14268 SDValue Val = isVMOVModifiedImm((~SplatBits).getZExtValue(),
14269 SplatUndef.getZExtValue(), SplatBitSize,
14270 DAG, dl, VbicVT, VT, OtherModImm);
14271 if (Val.getNode()) {
14272 SDValue Input =
14273 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0));
14274 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
14275 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic);
14276 }
14277 }
14278 }
14279
14280 if (!Subtarget->isThumb1Only()) {
14281 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
14282 if (SDValue Result = combineSelectAndUseCommutative(N, true, DCI))
14283 return Result;
14284
14285 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
14286 return Result;
14287 }
14288
14289 if (Subtarget->isThumb1Only())
14290 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
14291 return Result;
14292
14293 return SDValue();
14294 }
14295
14296 // Try combining OR nodes to SMULWB, SMULWT.
PerformORCombineToSMULWBT(SDNode * OR,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14297 static SDValue PerformORCombineToSMULWBT(SDNode *OR,
14298 TargetLowering::DAGCombinerInfo &DCI,
14299 const ARMSubtarget *Subtarget) {
14300 if (!Subtarget->hasV6Ops() ||
14301 (Subtarget->isThumb() &&
14302 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14303 return SDValue();
14304
14305 SDValue SRL = OR->getOperand(0);
14306 SDValue SHL = OR->getOperand(1);
14307
14308 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) {
14309 SRL = OR->getOperand(1);
14310 SHL = OR->getOperand(0);
14311 }
14312 if (!isSRL16(SRL) || !isSHL16(SHL))
14313 return SDValue();
14314
14315 // The first operands to the shifts need to be the two results from the
14316 // same smul_lohi node.
14317 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14318 SRL.getOperand(0).getOpcode() != ISD::SMUL_LOHI)
14319 return SDValue();
14320
14321 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14322 if (SRL.getOperand(0) != SDValue(SMULLOHI, 0) ||
14323 SHL.getOperand(0) != SDValue(SMULLOHI, 1))
14324 return SDValue();
14325
14326 // Now we have:
14327 // (or (srl (smul_lohi ?, ?), 16), (shl (smul_lohi ?, ?), 16)))
14328 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments.
14329 // For SMUWB the 16-bit value will signed extended somehow.
14330 // For SMULWT only the SRA is required.
14331 // Check both sides of SMUL_LOHI
14332 SDValue OpS16 = SMULLOHI->getOperand(0);
14333 SDValue OpS32 = SMULLOHI->getOperand(1);
14334
14335 SelectionDAG &DAG = DCI.DAG;
14336 if (!isS16(OpS16, DAG) && !isSRA16(OpS16)) {
14337 OpS16 = OpS32;
14338 OpS32 = SMULLOHI->getOperand(0);
14339 }
14340
14341 SDLoc dl(OR);
14342 unsigned Opcode = 0;
14343 if (isS16(OpS16, DAG))
14344 Opcode = ARMISD::SMULWB;
14345 else if (isSRA16(OpS16)) {
14346 Opcode = ARMISD::SMULWT;
14347 OpS16 = OpS16->getOperand(0);
14348 }
14349 else
14350 return SDValue();
14351
14352 SDValue Res = DAG.getNode(Opcode, dl, MVT::i32, OpS32, OpS16);
14353 DAG.ReplaceAllUsesOfValueWith(SDValue(OR, 0), Res);
14354 return SDValue(OR, 0);
14355 }
14356
PerformORCombineToBFI(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14357 static SDValue PerformORCombineToBFI(SDNode *N,
14358 TargetLowering::DAGCombinerInfo &DCI,
14359 const ARMSubtarget *Subtarget) {
14360 // BFI is only available on V6T2+
14361 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
14362 return SDValue();
14363
14364 EVT VT = N->getValueType(0);
14365 SDValue N0 = N->getOperand(0);
14366 SDValue N1 = N->getOperand(1);
14367 SelectionDAG &DAG = DCI.DAG;
14368 SDLoc DL(N);
14369 // 1) or (and A, mask), val => ARMbfi A, val, mask
14370 // iff (val & mask) == val
14371 //
14372 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14373 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
14374 // && mask == ~mask2
14375 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
14376 // && ~mask == mask2
14377 // (i.e., copy a bitfield value into another bitfield of the same width)
14378
14379 if (VT != MVT::i32)
14380 return SDValue();
14381
14382 SDValue N00 = N0.getOperand(0);
14383
14384 // The value and the mask need to be constants so we can verify this is
14385 // actually a bitfield set. If the mask is 0xffff, we can do better
14386 // via a movt instruction, so don't use BFI in that case.
14387 SDValue MaskOp = N0.getOperand(1);
14388 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(MaskOp);
14389 if (!MaskC)
14390 return SDValue();
14391 unsigned Mask = MaskC->getZExtValue();
14392 if (Mask == 0xffff)
14393 return SDValue();
14394 SDValue Res;
14395 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
14396 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
14397 if (N1C) {
14398 unsigned Val = N1C->getZExtValue();
14399 if ((Val & ~Mask) != Val)
14400 return SDValue();
14401
14402 if (ARM::isBitFieldInvertedMask(Mask)) {
14403 Val >>= countTrailingZeros(~Mask);
14404
14405 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00,
14406 DAG.getConstant(Val, DL, MVT::i32),
14407 DAG.getConstant(Mask, DL, MVT::i32));
14408
14409 DCI.CombineTo(N, Res, false);
14410 // Return value from the original node to inform the combiner than N is
14411 // now dead.
14412 return SDValue(N, 0);
14413 }
14414 } else if (N1.getOpcode() == ISD::AND) {
14415 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14416 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
14417 if (!N11C)
14418 return SDValue();
14419 unsigned Mask2 = N11C->getZExtValue();
14420
14421 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
14422 // as is to match.
14423 if (ARM::isBitFieldInvertedMask(Mask) &&
14424 (Mask == ~Mask2)) {
14425 // The pack halfword instruction works better for masks that fit it,
14426 // so use that when it's available.
14427 if (Subtarget->hasDSP() &&
14428 (Mask == 0xffff || Mask == 0xffff0000))
14429 return SDValue();
14430 // 2a
14431 unsigned amt = countTrailingZeros(Mask2);
14432 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0),
14433 DAG.getConstant(amt, DL, MVT::i32));
14434 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res,
14435 DAG.getConstant(Mask, DL, MVT::i32));
14436 DCI.CombineTo(N, Res, false);
14437 // Return value from the original node to inform the combiner than N is
14438 // now dead.
14439 return SDValue(N, 0);
14440 } else if (ARM::isBitFieldInvertedMask(~Mask) &&
14441 (~Mask == Mask2)) {
14442 // The pack halfword instruction works better for masks that fit it,
14443 // so use that when it's available.
14444 if (Subtarget->hasDSP() &&
14445 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14446 return SDValue();
14447 // 2b
14448 unsigned lsb = countTrailingZeros(Mask);
14449 Res = DAG.getNode(ISD::SRL, DL, VT, N00,
14450 DAG.getConstant(lsb, DL, MVT::i32));
14451 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res,
14452 DAG.getConstant(Mask2, DL, MVT::i32));
14453 DCI.CombineTo(N, Res, false);
14454 // Return value from the original node to inform the combiner than N is
14455 // now dead.
14456 return SDValue(N, 0);
14457 }
14458 }
14459
14460 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) &&
14461 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N00.getOperand(1)) &&
14462 ARM::isBitFieldInvertedMask(~Mask)) {
14463 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
14464 // where lsb(mask) == #shamt and masked bits of B are known zero.
14465 SDValue ShAmt = N00.getOperand(1);
14466 unsigned ShAmtC = cast<ConstantSDNode>(ShAmt)->getZExtValue();
14467 unsigned LSB = countTrailingZeros(Mask);
14468 if (ShAmtC != LSB)
14469 return SDValue();
14470
14471 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0),
14472 DAG.getConstant(~Mask, DL, MVT::i32));
14473
14474 DCI.CombineTo(N, Res, false);
14475 // Return value from the original node to inform the combiner than N is
14476 // now dead.
14477 return SDValue(N, 0);
14478 }
14479
14480 return SDValue();
14481 }
14482
isValidMVECond(unsigned CC,bool IsFloat)14483 static bool isValidMVECond(unsigned CC, bool IsFloat) {
14484 switch (CC) {
14485 case ARMCC::EQ:
14486 case ARMCC::NE:
14487 case ARMCC::LE:
14488 case ARMCC::GT:
14489 case ARMCC::GE:
14490 case ARMCC::LT:
14491 return true;
14492 case ARMCC::HS:
14493 case ARMCC::HI:
14494 return !IsFloat;
14495 default:
14496 return false;
14497 };
14498 }
14499
getVCMPCondCode(SDValue N)14500 static ARMCC::CondCodes getVCMPCondCode(SDValue N) {
14501 if (N->getOpcode() == ARMISD::VCMP)
14502 return (ARMCC::CondCodes)N->getConstantOperandVal(2);
14503 else if (N->getOpcode() == ARMISD::VCMPZ)
14504 return (ARMCC::CondCodes)N->getConstantOperandVal(1);
14505 else
14506 llvm_unreachable("Not a VCMP/VCMPZ!");
14507 }
14508
CanInvertMVEVCMP(SDValue N)14509 static bool CanInvertMVEVCMP(SDValue N) {
14510 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(getVCMPCondCode(N));
14511 return isValidMVECond(CC, N->getOperand(0).getValueType().isFloatingPoint());
14512 }
14513
PerformORCombine_i1(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)14514 static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG,
14515 const ARMSubtarget *Subtarget) {
14516 // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain
14517 // together with predicates
14518 EVT VT = N->getValueType(0);
14519 SDLoc DL(N);
14520 SDValue N0 = N->getOperand(0);
14521 SDValue N1 = N->getOperand(1);
14522
14523 auto IsFreelyInvertable = [&](SDValue V) {
14524 if (V->getOpcode() == ARMISD::VCMP || V->getOpcode() == ARMISD::VCMPZ)
14525 return CanInvertMVEVCMP(V);
14526 return false;
14527 };
14528
14529 // At least one operand must be freely invertable.
14530 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14531 return SDValue();
14532
14533 SDValue NewN0 = DAG.getLogicalNOT(DL, N0, VT);
14534 SDValue NewN1 = DAG.getLogicalNOT(DL, N1, VT);
14535 SDValue And = DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1);
14536 return DAG.getLogicalNOT(DL, And, VT);
14537 }
14538
14539 /// PerformORCombine - Target-specific dag combine xforms for ISD::OR
PerformORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14540 static SDValue PerformORCombine(SDNode *N,
14541 TargetLowering::DAGCombinerInfo &DCI,
14542 const ARMSubtarget *Subtarget) {
14543 // Attempt to use immediate-form VORR
14544 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
14545 SDLoc dl(N);
14546 EVT VT = N->getValueType(0);
14547 SelectionDAG &DAG = DCI.DAG;
14548
14549 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14550 return SDValue();
14551
14552 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14553 VT == MVT::v8i1 || VT == MVT::v16i1))
14554 return PerformORCombine_i1(N, DAG, Subtarget);
14555
14556 APInt SplatBits, SplatUndef;
14557 unsigned SplatBitSize;
14558 bool HasAnyUndefs;
14559 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14560 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14561 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14562 SplatBitSize == 64) {
14563 EVT VorrVT;
14564 SDValue Val =
14565 isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
14566 SplatBitSize, DAG, dl, VorrVT, VT, OtherModImm);
14567 if (Val.getNode()) {
14568 SDValue Input =
14569 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0));
14570 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
14571 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr);
14572 }
14573 }
14574 }
14575
14576 if (!Subtarget->isThumb1Only()) {
14577 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
14578 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
14579 return Result;
14580 if (SDValue Result = PerformORCombineToSMULWBT(N, DCI, Subtarget))
14581 return Result;
14582 }
14583
14584 SDValue N0 = N->getOperand(0);
14585 SDValue N1 = N->getOperand(1);
14586
14587 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
14588 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
14589 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
14590
14591 // The code below optimizes (or (and X, Y), Z).
14592 // The AND operand needs to have a single user to make these optimizations
14593 // profitable.
14594 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
14595 return SDValue();
14596
14597 APInt SplatUndef;
14598 unsigned SplatBitSize;
14599 bool HasAnyUndefs;
14600
14601 APInt SplatBits0, SplatBits1;
14602 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(1));
14603 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(1));
14604 // Ensure that the second operand of both ands are constants
14605 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14606 HasAnyUndefs) && !HasAnyUndefs) {
14607 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14608 HasAnyUndefs) && !HasAnyUndefs) {
14609 // Ensure that the bit width of the constants are the same and that
14610 // the splat arguments are logical inverses as per the pattern we
14611 // are trying to simplify.
14612 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
14613 SplatBits0 == ~SplatBits1) {
14614 // Canonicalize the vector type to make instruction selection
14615 // simpler.
14616 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
14617 SDValue Result = DAG.getNode(ARMISD::VBSP, dl, CanonicalVT,
14618 N0->getOperand(1),
14619 N0->getOperand(0),
14620 N1->getOperand(0));
14621 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
14622 }
14623 }
14624 }
14625 }
14626
14627 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
14628 // reasonable.
14629 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
14630 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
14631 return Res;
14632 }
14633
14634 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
14635 return Result;
14636
14637 return SDValue();
14638 }
14639
PerformXORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14640 static SDValue PerformXORCombine(SDNode *N,
14641 TargetLowering::DAGCombinerInfo &DCI,
14642 const ARMSubtarget *Subtarget) {
14643 EVT VT = N->getValueType(0);
14644 SelectionDAG &DAG = DCI.DAG;
14645
14646 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14647 return SDValue();
14648
14649 if (!Subtarget->isThumb1Only()) {
14650 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
14651 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
14652 return Result;
14653
14654 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
14655 return Result;
14656 }
14657
14658 if (Subtarget->hasMVEIntegerOps()) {
14659 // fold (xor(vcmp/z, 1)) into a vcmp with the opposite condition.
14660 SDValue N0 = N->getOperand(0);
14661 SDValue N1 = N->getOperand(1);
14662 const TargetLowering *TLI = Subtarget->getTargetLowering();
14663 if (TLI->isConstTrueVal(N1) &&
14664 (N0->getOpcode() == ARMISD::VCMP || N0->getOpcode() == ARMISD::VCMPZ)) {
14665 if (CanInvertMVEVCMP(N0)) {
14666 SDLoc DL(N0);
14667 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(getVCMPCondCode(N0));
14668
14669 SmallVector<SDValue, 4> Ops;
14670 Ops.push_back(N0->getOperand(0));
14671 if (N0->getOpcode() == ARMISD::VCMP)
14672 Ops.push_back(N0->getOperand(1));
14673 Ops.push_back(DAG.getConstant(CC, DL, MVT::i32));
14674 return DAG.getNode(N0->getOpcode(), DL, N0->getValueType(0), Ops);
14675 }
14676 }
14677 }
14678
14679 return SDValue();
14680 }
14681
14682 // ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
14683 // and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
14684 // their position in "to" (Rd).
ParseBFI(SDNode * N,APInt & ToMask,APInt & FromMask)14685 static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
14686 assert(N->getOpcode() == ARMISD::BFI);
14687
14688 SDValue From = N->getOperand(1);
14689 ToMask = ~cast<ConstantSDNode>(N->getOperand(2))->getAPIntValue();
14690 FromMask = APInt::getLowBitsSet(ToMask.getBitWidth(), ToMask.countPopulation());
14691
14692 // If the Base came from a SHR #C, we can deduce that it is really testing bit
14693 // #C in the base of the SHR.
14694 if (From->getOpcode() == ISD::SRL &&
14695 isa<ConstantSDNode>(From->getOperand(1))) {
14696 APInt Shift = cast<ConstantSDNode>(From->getOperand(1))->getAPIntValue();
14697 assert(Shift.getLimitedValue() < 32 && "Shift too large!");
14698 FromMask <<= Shift.getLimitedValue(31);
14699 From = From->getOperand(0);
14700 }
14701
14702 return From;
14703 }
14704
14705 // If A and B contain one contiguous set of bits, does A | B == A . B?
14706 //
14707 // Neither A nor B must be zero.
BitsProperlyConcatenate(const APInt & A,const APInt & B)14708 static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
14709 unsigned LastActiveBitInA = A.countTrailingZeros();
14710 unsigned FirstActiveBitInB = B.getBitWidth() - B.countLeadingZeros() - 1;
14711 return LastActiveBitInA - 1 == FirstActiveBitInB;
14712 }
14713
FindBFIToCombineWith(SDNode * N)14714 static SDValue FindBFIToCombineWith(SDNode *N) {
14715 // We have a BFI in N. Find a BFI it can combine with, if one exists.
14716 APInt ToMask, FromMask;
14717 SDValue From = ParseBFI(N, ToMask, FromMask);
14718 SDValue To = N->getOperand(0);
14719
14720 SDValue V = To;
14721 if (V.getOpcode() != ARMISD::BFI)
14722 return SDValue();
14723
14724 APInt NewToMask, NewFromMask;
14725 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask);
14726 if (NewFrom != From)
14727 return SDValue();
14728
14729 // Do the written bits conflict with any we've seen so far?
14730 if ((NewToMask & ToMask).getBoolValue())
14731 // Conflicting bits.
14732 return SDValue();
14733
14734 // Are the new bits contiguous when combined with the old bits?
14735 if (BitsProperlyConcatenate(ToMask, NewToMask) &&
14736 BitsProperlyConcatenate(FromMask, NewFromMask))
14737 return V;
14738 if (BitsProperlyConcatenate(NewToMask, ToMask) &&
14739 BitsProperlyConcatenate(NewFromMask, FromMask))
14740 return V;
14741
14742 return SDValue();
14743 }
14744
PerformBFICombine(SDNode * N,SelectionDAG & DAG)14745 static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG) {
14746 SDValue N0 = N->getOperand(0);
14747 SDValue N1 = N->getOperand(1);
14748
14749 if (N1.getOpcode() == ISD::AND) {
14750 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
14751 // the bits being cleared by the AND are not demanded by the BFI.
14752 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
14753 if (!N11C)
14754 return SDValue();
14755 unsigned InvMask = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
14756 unsigned LSB = countTrailingZeros(~InvMask);
14757 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14758 assert(Width <
14759 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14760 "undefined behavior");
14761 unsigned Mask = (1u << Width) - 1;
14762 unsigned Mask2 = N11C->getZExtValue();
14763 if ((Mask & (~Mask2)) == 0)
14764 return DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0),
14765 N->getOperand(0), N1.getOperand(0), N->getOperand(2));
14766 return SDValue();
14767 }
14768
14769 // Look for another BFI to combine with.
14770 if (SDValue CombineBFI = FindBFIToCombineWith(N)) {
14771 // We've found a BFI.
14772 APInt ToMask1, FromMask1;
14773 SDValue From1 = ParseBFI(N, ToMask1, FromMask1);
14774
14775 APInt ToMask2, FromMask2;
14776 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2);
14777 assert(From1 == From2);
14778 (void)From2;
14779
14780 // Create a new BFI, combining the two together.
14781 APInt NewFromMask = FromMask1 | FromMask2;
14782 APInt NewToMask = ToMask1 | ToMask2;
14783
14784 EVT VT = N->getValueType(0);
14785 SDLoc dl(N);
14786
14787 if (NewFromMask[0] == 0)
14788 From1 = DAG.getNode(
14789 ISD::SRL, dl, VT, From1,
14790 DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT));
14791 return DAG.getNode(ARMISD::BFI, dl, VT, CombineBFI.getOperand(0), From1,
14792 DAG.getConstant(~NewToMask, dl, VT));
14793 }
14794
14795 // Reassociate BFI(BFI (A, B, M1), C, M2) to BFI(BFI (A, C, M2), B, M1) so
14796 // that lower bit insertions are performed first, providing that M1 and M2
14797 // do no overlap. This can allow multiple BFI instructions to be combined
14798 // together by the other folds above.
14799 if (N->getOperand(0).getOpcode() == ARMISD::BFI) {
14800 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14801 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14802
14803 if (!N0.hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14804 ToMask1.countLeadingZeros() < ToMask2.countLeadingZeros())
14805 return SDValue();
14806
14807 EVT VT = N->getValueType(0);
14808 SDLoc dl(N);
14809 SDValue BFI1 = DAG.getNode(ARMISD::BFI, dl, VT, N0.getOperand(0),
14810 N->getOperand(1), N->getOperand(2));
14811 return DAG.getNode(ARMISD::BFI, dl, VT, BFI1, N0.getOperand(1),
14812 N0.getOperand(2));
14813 }
14814
14815 return SDValue();
14816 }
14817
14818 // Check that N is CMPZ(CSINC(0, 0, CC, X)),
14819 // or CMPZ(CMOV(1, 0, CC, $cpsr, X))
14820 // return X if valid.
IsCMPZCSINC(SDNode * Cmp,ARMCC::CondCodes & CC)14821 static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) {
14822 if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(Cmp->getOperand(1)))
14823 return SDValue();
14824 SDValue CSInc = Cmp->getOperand(0);
14825
14826 // Ignore any `And 1` nodes that may not yet have been removed. We are
14827 // looking for a value that produces 1/0, so these have no effect on the
14828 // code.
14829 while (CSInc.getOpcode() == ISD::AND &&
14830 isa<ConstantSDNode>(CSInc.getOperand(1)) &&
14831 CSInc.getConstantOperandVal(1) == 1 && CSInc->hasOneUse())
14832 CSInc = CSInc.getOperand(0);
14833
14834 if (CSInc.getOpcode() == ARMISD::CSINC &&
14835 isNullConstant(CSInc.getOperand(0)) &&
14836 isNullConstant(CSInc.getOperand(1)) && CSInc->hasOneUse()) {
14837 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(2);
14838 return CSInc.getOperand(3);
14839 }
14840 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(CSInc.getOperand(0)) &&
14841 isNullConstant(CSInc.getOperand(1)) && CSInc->hasOneUse()) {
14842 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(2);
14843 return CSInc.getOperand(4);
14844 }
14845 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(CSInc.getOperand(1)) &&
14846 isNullConstant(CSInc.getOperand(0)) && CSInc->hasOneUse()) {
14847 CC = ARMCC::getOppositeCondition(
14848 (ARMCC::CondCodes)CSInc.getConstantOperandVal(2));
14849 return CSInc.getOperand(4);
14850 }
14851 return SDValue();
14852 }
14853
PerformCMPZCombine(SDNode * N,SelectionDAG & DAG)14854 static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG) {
14855 // Given CMPZ(CSINC(C, 0, 0, EQ), 0), we can just use C directly. As in
14856 // t92: glue = ARMISD::CMPZ t74, 0
14857 // t93: i32 = ARMISD::CSINC 0, 0, 1, t92
14858 // t96: glue = ARMISD::CMPZ t93, 0
14859 // t114: i32 = ARMISD::CSINV 0, 0, 0, t96
14860 ARMCC::CondCodes Cond;
14861 if (SDValue C = IsCMPZCSINC(N, Cond))
14862 if (Cond == ARMCC::EQ)
14863 return C;
14864 return SDValue();
14865 }
14866
PerformCSETCombine(SDNode * N,SelectionDAG & DAG)14867 static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG) {
14868 // Fold away an unneccessary CMPZ/CSINC
14869 // CSXYZ A, B, C1 (CMPZ (CSINC 0, 0, C2, D), 0) ->
14870 // if C1==EQ -> CSXYZ A, B, C2, D
14871 // if C1==NE -> CSXYZ A, B, NOT(C2), D
14872 ARMCC::CondCodes Cond;
14873 if (SDValue C = IsCMPZCSINC(N->getOperand(3).getNode(), Cond)) {
14874 if (N->getConstantOperandVal(2) == ARMCC::EQ)
14875 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
14876 N->getOperand(1),
14877 DAG.getConstant(Cond, SDLoc(N), MVT::i32), C);
14878 if (N->getConstantOperandVal(2) == ARMCC::NE)
14879 return DAG.getNode(
14880 N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
14881 N->getOperand(1),
14882 DAG.getConstant(ARMCC::getOppositeCondition(Cond), SDLoc(N), MVT::i32), C);
14883 }
14884 return SDValue();
14885 }
14886
14887 /// PerformVMOVRRDCombine - Target-specific dag combine xforms for
14888 /// ARMISD::VMOVRRD.
PerformVMOVRRDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)14889 static SDValue PerformVMOVRRDCombine(SDNode *N,
14890 TargetLowering::DAGCombinerInfo &DCI,
14891 const ARMSubtarget *Subtarget) {
14892 // vmovrrd(vmovdrr x, y) -> x,y
14893 SDValue InDouble = N->getOperand(0);
14894 if (InDouble.getOpcode() == ARMISD::VMOVDRR && Subtarget->hasFP64())
14895 return DCI.CombineTo(N, InDouble.getOperand(0), InDouble.getOperand(1));
14896
14897 // vmovrrd(load f64) -> (load i32), (load i32)
14898 SDNode *InNode = InDouble.getNode();
14899 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() &&
14900 InNode->getValueType(0) == MVT::f64 &&
14901 InNode->getOperand(1).getOpcode() == ISD::FrameIndex &&
14902 !cast<LoadSDNode>(InNode)->isVolatile()) {
14903 // TODO: Should this be done for non-FrameIndex operands?
14904 LoadSDNode *LD = cast<LoadSDNode>(InNode);
14905
14906 SelectionDAG &DAG = DCI.DAG;
14907 SDLoc DL(LD);
14908 SDValue BasePtr = LD->getBasePtr();
14909 SDValue NewLD1 =
14910 DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
14911 LD->getAlign(), LD->getMemOperand()->getFlags());
14912
14913 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
14914 DAG.getConstant(4, DL, MVT::i32));
14915
14916 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, LD->getChain(), OffsetPtr,
14917 LD->getPointerInfo().getWithOffset(4),
14918 commonAlignment(LD->getAlign(), 4),
14919 LD->getMemOperand()->getFlags());
14920
14921 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
14922 if (DCI.DAG.getDataLayout().isBigEndian())
14923 std::swap (NewLD1, NewLD2);
14924 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
14925 return Result;
14926 }
14927
14928 // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d
14929 // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b
14930 if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
14931 isa<ConstantSDNode>(InDouble.getOperand(1))) {
14932 SDValue BV = InDouble.getOperand(0);
14933 // Look up through any nop bitcasts and vector_reg_casts. bitcasts may
14934 // change lane order under big endian.
14935 bool BVSwap = BV.getOpcode() == ISD::BITCAST;
14936 while (
14937 (BV.getOpcode() == ISD::BITCAST ||
14938 BV.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
14939 (BV.getValueType() == MVT::v2f64 || BV.getValueType() == MVT::v2i64)) {
14940 BVSwap = BV.getOpcode() == ISD::BITCAST;
14941 BV = BV.getOperand(0);
14942 }
14943 if (BV.getValueType() != MVT::v4i32)
14944 return SDValue();
14945
14946 // Handle buildvectors, pulling out the correct lane depending on
14947 // endianness.
14948 unsigned Offset = InDouble.getConstantOperandVal(1) == 1 ? 2 : 0;
14949 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
14950 SDValue Op0 = BV.getOperand(Offset);
14951 SDValue Op1 = BV.getOperand(Offset + 1);
14952 if (!Subtarget->isLittle() && BVSwap)
14953 std::swap(Op0, Op1);
14954
14955 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
14956 }
14957
14958 // A chain of insert_vectors, grabbing the correct value of the chain of
14959 // inserts.
14960 SDValue Op0, Op1;
14961 while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) {
14962 if (isa<ConstantSDNode>(BV.getOperand(2))) {
14963 if (BV.getConstantOperandVal(2) == Offset)
14964 Op0 = BV.getOperand(1);
14965 if (BV.getConstantOperandVal(2) == Offset + 1)
14966 Op1 = BV.getOperand(1);
14967 }
14968 BV = BV.getOperand(0);
14969 }
14970 if (!Subtarget->isLittle() && BVSwap)
14971 std::swap(Op0, Op1);
14972 if (Op0 && Op1)
14973 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
14974 }
14975
14976 return SDValue();
14977 }
14978
14979 /// PerformVMOVDRRCombine - Target-specific dag combine xforms for
14980 /// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
PerformVMOVDRRCombine(SDNode * N,SelectionDAG & DAG)14981 static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
14982 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
14983 SDValue Op0 = N->getOperand(0);
14984 SDValue Op1 = N->getOperand(1);
14985 if (Op0.getOpcode() == ISD::BITCAST)
14986 Op0 = Op0.getOperand(0);
14987 if (Op1.getOpcode() == ISD::BITCAST)
14988 Op1 = Op1.getOperand(0);
14989 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
14990 Op0.getNode() == Op1.getNode() &&
14991 Op0.getResNo() == 0 && Op1.getResNo() == 1)
14992 return DAG.getNode(ISD::BITCAST, SDLoc(N),
14993 N->getValueType(0), Op0.getOperand(0));
14994 return SDValue();
14995 }
14996
PerformVMOVhrCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)14997 static SDValue PerformVMOVhrCombine(SDNode *N,
14998 TargetLowering::DAGCombinerInfo &DCI) {
14999 SDValue Op0 = N->getOperand(0);
15000
15001 // VMOVhr (VMOVrh (X)) -> X
15002 if (Op0->getOpcode() == ARMISD::VMOVrh)
15003 return Op0->getOperand(0);
15004
15005 // FullFP16: half values are passed in S-registers, and we don't
15006 // need any of the bitcast and moves:
15007 //
15008 // t2: f32,ch = CopyFromReg t0, Register:f32 %0
15009 // t5: i32 = bitcast t2
15010 // t18: f16 = ARMISD::VMOVhr t5
15011 if (Op0->getOpcode() == ISD::BITCAST) {
15012 SDValue Copy = Op0->getOperand(0);
15013 if (Copy.getValueType() == MVT::f32 &&
15014 Copy->getOpcode() == ISD::CopyFromReg) {
15015 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1)};
15016 SDValue NewCopy =
15017 DCI.DAG.getNode(ISD::CopyFromReg, SDLoc(N), N->getValueType(0), Ops);
15018 return NewCopy;
15019 }
15020 }
15021
15022 // fold (VMOVhr (load x)) -> (load (f16*)x)
15023 if (LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15024 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15025 LN0->getMemoryVT() == MVT::i16) {
15026 SDValue Load =
15027 DCI.DAG.getLoad(N->getValueType(0), SDLoc(N), LN0->getChain(),
15028 LN0->getBasePtr(), LN0->getMemOperand());
15029 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
15030 DCI.DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Load.getValue(1));
15031 return Load;
15032 }
15033 }
15034
15035 // Only the bottom 16 bits of the source register are used.
15036 APInt DemandedMask = APInt::getLowBitsSet(32, 16);
15037 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15038 if (TLI.SimplifyDemandedBits(Op0, DemandedMask, DCI))
15039 return SDValue(N, 0);
15040
15041 return SDValue();
15042 }
15043
PerformVMOVrhCombine(SDNode * N,SelectionDAG & DAG)15044 static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG) {
15045 SDValue N0 = N->getOperand(0);
15046 EVT VT = N->getValueType(0);
15047
15048 // fold (VMOVrh (fpconst x)) -> const x
15049 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N0)) {
15050 APFloat V = C->getValueAPF();
15051 return DAG.getConstant(V.bitcastToAPInt().getZExtValue(), SDLoc(N), VT);
15052 }
15053
15054 // fold (VMOVrh (load x)) -> (zextload (i16*)x)
15055 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse()) {
15056 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
15057
15058 SDValue Load =
15059 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(),
15060 LN0->getBasePtr(), MVT::i16, LN0->getMemOperand());
15061 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
15062 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
15063 return Load;
15064 }
15065
15066 // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n)
15067 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15068 isa<ConstantSDNode>(N0->getOperand(1)))
15069 return DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0),
15070 N0->getOperand(1));
15071
15072 return SDValue();
15073 }
15074
15075 /// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
15076 /// are normal, non-volatile loads. If so, it is profitable to bitcast an
15077 /// i64 vector to have f64 elements, since the value can then be loaded
15078 /// directly into a VFP register.
hasNormalLoadOperand(SDNode * N)15079 static bool hasNormalLoadOperand(SDNode *N) {
15080 unsigned NumElts = N->getValueType(0).getVectorNumElements();
15081 for (unsigned i = 0; i < NumElts; ++i) {
15082 SDNode *Elt = N->getOperand(i).getNode();
15083 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile())
15084 return true;
15085 }
15086 return false;
15087 }
15088
15089 /// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
15090 /// ISD::BUILD_VECTOR.
PerformBUILD_VECTORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)15091 static SDValue PerformBUILD_VECTORCombine(SDNode *N,
15092 TargetLowering::DAGCombinerInfo &DCI,
15093 const ARMSubtarget *Subtarget) {
15094 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
15095 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
15096 // into a pair of GPRs, which is fine when the value is used as a scalar,
15097 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
15098 SelectionDAG &DAG = DCI.DAG;
15099 if (N->getNumOperands() == 2)
15100 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
15101 return RV;
15102
15103 // Load i64 elements as f64 values so that type legalization does not split
15104 // them up into i32 values.
15105 EVT VT = N->getValueType(0);
15106 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
15107 return SDValue();
15108 SDLoc dl(N);
15109 SmallVector<SDValue, 8> Ops;
15110 unsigned NumElts = VT.getVectorNumElements();
15111 for (unsigned i = 0; i < NumElts; ++i) {
15112 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
15113 Ops.push_back(V);
15114 // Make the DAGCombiner fold the bitcast.
15115 DCI.AddToWorklist(V.getNode());
15116 }
15117 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, NumElts);
15118 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops);
15119 return DAG.getNode(ISD::BITCAST, dl, VT, BV);
15120 }
15121
15122 /// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
15123 static SDValue
PerformARMBUILD_VECTORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)15124 PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15125 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
15126 // At that time, we may have inserted bitcasts from integer to float.
15127 // If these bitcasts have survived DAGCombine, change the lowering of this
15128 // BUILD_VECTOR in something more vector friendly, i.e., that does not
15129 // force to use floating point types.
15130
15131 // Make sure we can change the type of the vector.
15132 // This is possible iff:
15133 // 1. The vector is only used in a bitcast to a integer type. I.e.,
15134 // 1.1. Vector is used only once.
15135 // 1.2. Use is a bit convert to an integer type.
15136 // 2. The size of its operands are 32-bits (64-bits are not legal).
15137 EVT VT = N->getValueType(0);
15138 EVT EltVT = VT.getVectorElementType();
15139
15140 // Check 1.1. and 2.
15141 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
15142 return SDValue();
15143
15144 // By construction, the input type must be float.
15145 assert(EltVT == MVT::f32 && "Unexpected type!");
15146
15147 // Check 1.2.
15148 SDNode *Use = *N->use_begin();
15149 if (Use->getOpcode() != ISD::BITCAST ||
15150 Use->getValueType(0).isFloatingPoint())
15151 return SDValue();
15152
15153 // Check profitability.
15154 // Model is, if more than half of the relevant operands are bitcast from
15155 // i32, turn the build_vector into a sequence of insert_vector_elt.
15156 // Relevant operands are everything that is not statically
15157 // (i.e., at compile time) bitcasted.
15158 unsigned NumOfBitCastedElts = 0;
15159 unsigned NumElts = VT.getVectorNumElements();
15160 unsigned NumOfRelevantElts = NumElts;
15161 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
15162 SDValue Elt = N->getOperand(Idx);
15163 if (Elt->getOpcode() == ISD::BITCAST) {
15164 // Assume only bit cast to i32 will go away.
15165 if (Elt->getOperand(0).getValueType() == MVT::i32)
15166 ++NumOfBitCastedElts;
15167 } else if (Elt.isUndef() || isa<ConstantSDNode>(Elt))
15168 // Constants are statically casted, thus do not count them as
15169 // relevant operands.
15170 --NumOfRelevantElts;
15171 }
15172
15173 // Check if more than half of the elements require a non-free bitcast.
15174 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15175 return SDValue();
15176
15177 SelectionDAG &DAG = DCI.DAG;
15178 // Create the new vector type.
15179 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
15180 // Check if the type is legal.
15181 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15182 if (!TLI.isTypeLegal(VecVT))
15183 return SDValue();
15184
15185 // Combine:
15186 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
15187 // => BITCAST INSERT_VECTOR_ELT
15188 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
15189 // (BITCAST EN), N.
15190 SDValue Vec = DAG.getUNDEF(VecVT);
15191 SDLoc dl(N);
15192 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
15193 SDValue V = N->getOperand(Idx);
15194 if (V.isUndef())
15195 continue;
15196 if (V.getOpcode() == ISD::BITCAST &&
15197 V->getOperand(0).getValueType() == MVT::i32)
15198 // Fold obvious case.
15199 V = V.getOperand(0);
15200 else {
15201 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V);
15202 // Make the DAGCombiner fold the bitcasts.
15203 DCI.AddToWorklist(V.getNode());
15204 }
15205 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
15206 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
15207 }
15208 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
15209 // Make the DAGCombiner fold the bitcasts.
15210 DCI.AddToWorklist(Vec.getNode());
15211 return Vec;
15212 }
15213
15214 static SDValue
PerformPREDICATE_CASTCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)15215 PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15216 EVT VT = N->getValueType(0);
15217 SDValue Op = N->getOperand(0);
15218 SDLoc dl(N);
15219
15220 // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
15221 if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
15222 // If the valuetypes are the same, we can remove the cast entirely.
15223 if (Op->getOperand(0).getValueType() == VT)
15224 return Op->getOperand(0);
15225 return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
15226 }
15227
15228 // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce
15229 // more VPNOT which might get folded as else predicates.
15230 if (Op.getValueType() == MVT::i32 && isBitwiseNot(Op)) {
15231 SDValue X =
15232 DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
15233 SDValue C = DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
15234 DCI.DAG.getConstant(65535, dl, MVT::i32));
15235 return DCI.DAG.getNode(ISD::XOR, dl, VT, X, C);
15236 }
15237
15238 // Only the bottom 16 bits of the source register are used.
15239 if (Op.getValueType() == MVT::i32) {
15240 APInt DemandedMask = APInt::getLowBitsSet(32, 16);
15241 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15242 if (TLI.SimplifyDemandedBits(Op, DemandedMask, DCI))
15243 return SDValue(N, 0);
15244 }
15245 return SDValue();
15246 }
15247
PerformVECTOR_REG_CASTCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)15248 static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG,
15249 const ARMSubtarget *ST) {
15250 EVT VT = N->getValueType(0);
15251 SDValue Op = N->getOperand(0);
15252 SDLoc dl(N);
15253
15254 // Under Little endian, a VECTOR_REG_CAST is equivalent to a BITCAST
15255 if (ST->isLittle())
15256 return DAG.getNode(ISD::BITCAST, dl, VT, Op);
15257
15258 // VECTOR_REG_CAST undef -> undef
15259 if (Op.isUndef())
15260 return DAG.getUNDEF(VT);
15261
15262 // VECTOR_REG_CAST(VECTOR_REG_CAST(x)) == VECTOR_REG_CAST(x)
15263 if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) {
15264 // If the valuetypes are the same, we can remove the cast entirely.
15265 if (Op->getOperand(0).getValueType() == VT)
15266 return Op->getOperand(0);
15267 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0));
15268 }
15269
15270 return SDValue();
15271 }
15272
PerformVCMPCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)15273 static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG,
15274 const ARMSubtarget *Subtarget) {
15275 if (!Subtarget->hasMVEIntegerOps())
15276 return SDValue();
15277
15278 EVT VT = N->getValueType(0);
15279 SDValue Op0 = N->getOperand(0);
15280 SDValue Op1 = N->getOperand(1);
15281 ARMCC::CondCodes Cond =
15282 (ARMCC::CondCodes)cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
15283 SDLoc dl(N);
15284
15285 // vcmp X, 0, cc -> vcmpz X, cc
15286 if (isZeroVector(Op1))
15287 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0, N->getOperand(2));
15288
15289 unsigned SwappedCond = getSwappedCondition(Cond);
15290 if (isValidMVECond(SwappedCond, VT.isFloatingPoint())) {
15291 // vcmp 0, X, cc -> vcmpz X, reversed(cc)
15292 if (isZeroVector(Op0))
15293 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1,
15294 DAG.getConstant(SwappedCond, dl, MVT::i32));
15295 // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc)
15296 if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP)
15297 return DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0,
15298 DAG.getConstant(SwappedCond, dl, MVT::i32));
15299 }
15300
15301 return SDValue();
15302 }
15303
15304 /// PerformInsertEltCombine - Target-specific dag combine xforms for
15305 /// ISD::INSERT_VECTOR_ELT.
PerformInsertEltCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)15306 static SDValue PerformInsertEltCombine(SDNode *N,
15307 TargetLowering::DAGCombinerInfo &DCI) {
15308 // Bitcast an i64 load inserted into a vector to f64.
15309 // Otherwise, the i64 value will be legalized to a pair of i32 values.
15310 EVT VT = N->getValueType(0);
15311 SDNode *Elt = N->getOperand(1).getNode();
15312 if (VT.getVectorElementType() != MVT::i64 ||
15313 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile())
15314 return SDValue();
15315
15316 SelectionDAG &DAG = DCI.DAG;
15317 SDLoc dl(N);
15318 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
15319 VT.getVectorNumElements());
15320 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
15321 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
15322 // Make the DAGCombiner fold the bitcasts.
15323 DCI.AddToWorklist(Vec.getNode());
15324 DCI.AddToWorklist(V.getNode());
15325 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
15326 Vec, V, N->getOperand(2));
15327 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt);
15328 }
15329
15330 // Convert a pair of extracts from the same base vector to a VMOVRRD. Either
15331 // directly or bitcast to an integer if the original is a float vector.
15332 // extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2)
15333 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2)
15334 static SDValue
PerformExtractEltToVMOVRRD(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)15335 PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15336 EVT VT = N->getValueType(0);
15337 SDLoc dl(N);
15338
15339 if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 ||
15340 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(MVT::f64))
15341 return SDValue();
15342
15343 SDValue Ext = SDValue(N, 0);
15344 if (Ext.getOpcode() == ISD::BITCAST &&
15345 Ext.getOperand(0).getValueType() == MVT::f32)
15346 Ext = Ext.getOperand(0);
15347 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
15348 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15349 Ext.getConstantOperandVal(1) % 2 != 0)
15350 return SDValue();
15351 if (Ext->use_size() == 1 &&
15352 (Ext->use_begin()->getOpcode() == ISD::SINT_TO_FP ||
15353 Ext->use_begin()->getOpcode() == ISD::UINT_TO_FP))
15354 return SDValue();
15355
15356 SDValue Op0 = Ext.getOperand(0);
15357 EVT VecVT = Op0.getValueType();
15358 unsigned ResNo = Op0.getResNo();
15359 unsigned Lane = Ext.getConstantOperandVal(1);
15360 if (VecVT.getVectorNumElements() != 4)
15361 return SDValue();
15362
15363 // Find another extract, of Lane + 1
15364 auto OtherIt = find_if(Op0->uses(), [&](SDNode *V) {
15365 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15366 isa<ConstantSDNode>(V->getOperand(1)) &&
15367 V->getConstantOperandVal(1) == Lane + 1 &&
15368 V->getOperand(0).getResNo() == ResNo;
15369 });
15370 if (OtherIt == Op0->uses().end())
15371 return SDValue();
15372
15373 // For float extracts, we need to be converting to a i32 for both vector
15374 // lanes.
15375 SDValue OtherExt(*OtherIt, 0);
15376 if (OtherExt.getValueType() != MVT::i32) {
15377 if (OtherExt->use_size() != 1 ||
15378 OtherExt->use_begin()->getOpcode() != ISD::BITCAST ||
15379 OtherExt->use_begin()->getValueType(0) != MVT::i32)
15380 return SDValue();
15381 OtherExt = SDValue(*OtherExt->use_begin(), 0);
15382 }
15383
15384 // Convert the type to a f64 and extract with a VMOVRRD.
15385 SDValue F64 = DCI.DAG.getNode(
15386 ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
15387 DCI.DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v2f64, Op0),
15388 DCI.DAG.getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15389 SDValue VMOVRRD =
15390 DCI.DAG.getNode(ARMISD::VMOVRRD, dl, {MVT::i32, MVT::i32}, F64);
15391
15392 DCI.CombineTo(OtherExt.getNode(), SDValue(VMOVRRD.getNode(), 1));
15393 return VMOVRRD;
15394 }
15395
PerformExtractEltCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * ST)15396 static SDValue PerformExtractEltCombine(SDNode *N,
15397 TargetLowering::DAGCombinerInfo &DCI,
15398 const ARMSubtarget *ST) {
15399 SDValue Op0 = N->getOperand(0);
15400 EVT VT = N->getValueType(0);
15401 SDLoc dl(N);
15402
15403 // extract (vdup x) -> x
15404 if (Op0->getOpcode() == ARMISD::VDUP) {
15405 SDValue X = Op0->getOperand(0);
15406 if (VT == MVT::f16 && X.getValueType() == MVT::i32)
15407 return DCI.DAG.getNode(ARMISD::VMOVhr, dl, VT, X);
15408 if (VT == MVT::i32 && X.getValueType() == MVT::f16)
15409 return DCI.DAG.getNode(ARMISD::VMOVrh, dl, VT, X);
15410 if (VT == MVT::f32 && X.getValueType() == MVT::i32)
15411 return DCI.DAG.getNode(ISD::BITCAST, dl, VT, X);
15412
15413 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST)
15414 X = X->getOperand(0);
15415 if (X.getValueType() == VT)
15416 return X;
15417 }
15418
15419 // extract ARM_BUILD_VECTOR -> x
15420 if (Op0->getOpcode() == ARMISD::BUILD_VECTOR &&
15421 isa<ConstantSDNode>(N->getOperand(1)) &&
15422 N->getConstantOperandVal(1) < Op0.getNumOperands()) {
15423 return Op0.getOperand(N->getConstantOperandVal(1));
15424 }
15425
15426 // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b
15427 if (Op0.getValueType() == MVT::v4i32 &&
15428 isa<ConstantSDNode>(N->getOperand(1)) &&
15429 Op0.getOpcode() == ISD::BITCAST &&
15430 Op0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
15431 Op0.getOperand(0).getValueType() == MVT::v2f64) {
15432 SDValue BV = Op0.getOperand(0);
15433 unsigned Offset = N->getConstantOperandVal(1);
15434 SDValue MOV = BV.getOperand(Offset < 2 ? 0 : 1);
15435 if (MOV.getOpcode() == ARMISD::VMOVDRR)
15436 return MOV.getOperand(ST->isLittle() ? Offset % 2 : 1 - Offset % 2);
15437 }
15438
15439 // extract x, n; extract x, n+1 -> VMOVRRD x
15440 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
15441 return R;
15442
15443 // extract (MVETrunc(x)) -> extract x
15444 if (Op0->getOpcode() == ARMISD::MVETRUNC) {
15445 unsigned Idx = N->getConstantOperandVal(1);
15446 unsigned Vec =
15447 Idx / Op0->getOperand(0).getValueType().getVectorNumElements();
15448 unsigned SubIdx =
15449 Idx % Op0->getOperand(0).getValueType().getVectorNumElements();
15450 return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Op0.getOperand(Vec),
15451 DCI.DAG.getConstant(SubIdx, dl, MVT::i32));
15452 }
15453
15454 return SDValue();
15455 }
15456
PerformSignExtendInregCombine(SDNode * N,SelectionDAG & DAG)15457 static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG) {
15458 SDValue Op = N->getOperand(0);
15459 EVT VT = N->getValueType(0);
15460
15461 // sext_inreg(VGETLANEu) -> VGETLANEs
15462 if (Op.getOpcode() == ARMISD::VGETLANEu &&
15463 cast<VTSDNode>(N->getOperand(1))->getVT() ==
15464 Op.getOperand(0).getValueType().getScalarType())
15465 return DAG.getNode(ARMISD::VGETLANEs, SDLoc(N), VT, Op.getOperand(0),
15466 Op.getOperand(1));
15467
15468 return SDValue();
15469 }
15470
15471 // When lowering complex nodes that we recognize, like VQDMULH and MULH, we
15472 // can end up with shuffle(binop(shuffle, shuffle)), that can be simplified to
15473 // binop as the shuffles cancel out.
FlattenVectorShuffle(ShuffleVectorSDNode * N,SelectionDAG & DAG)15474 static SDValue FlattenVectorShuffle(ShuffleVectorSDNode *N, SelectionDAG &DAG) {
15475 EVT VT = N->getValueType(0);
15476 if (!N->getOperand(1).isUndef() || N->getOperand(0).getValueType() != VT)
15477 return SDValue();
15478 SDValue Op = N->getOperand(0);
15479
15480 // Looking for binary operators that will have been folded from
15481 // truncates/extends.
15482 switch (Op.getOpcode()) {
15483 case ARMISD::VQDMULH:
15484 case ISD::MULHS:
15485 case ISD::MULHU:
15486 case ISD::ABDS:
15487 case ISD::ABDU:
15488 case ISD::AVGFLOORS:
15489 case ISD::AVGFLOORU:
15490 case ISD::AVGCEILS:
15491 case ISD::AVGCEILU:
15492 break;
15493 default:
15494 return SDValue();
15495 }
15496
15497 ShuffleVectorSDNode *Op0 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(0));
15498 ShuffleVectorSDNode *Op1 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(1));
15499 if (!Op0 || !Op1 || !Op0->getOperand(1).isUndef() ||
15500 !Op1->getOperand(1).isUndef() || Op0->getMask() != Op1->getMask() ||
15501 Op0->getOperand(0).getValueType() != VT)
15502 return SDValue();
15503
15504 // Check the mask turns into an identity shuffle.
15505 ArrayRef<int> NMask = N->getMask();
15506 ArrayRef<int> OpMask = Op0->getMask();
15507 for (int i = 0, e = NMask.size(); i != e; i++) {
15508 if (NMask[i] > 0 && OpMask[NMask[i]] > 0 && OpMask[NMask[i]] != i)
15509 return SDValue();
15510 }
15511
15512 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
15513 Op0->getOperand(0), Op1->getOperand(0));
15514 }
15515
15516 static SDValue
PerformInsertSubvectorCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)15517 PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15518 SDValue Vec = N->getOperand(0);
15519 SDValue SubVec = N->getOperand(1);
15520 uint64_t IdxVal = N->getConstantOperandVal(2);
15521 EVT VecVT = Vec.getValueType();
15522 EVT SubVT = SubVec.getValueType();
15523
15524 // Only do this for legal fixed vector types.
15525 if (!VecVT.isFixedLengthVector() ||
15526 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VecVT) ||
15527 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(SubVT))
15528 return SDValue();
15529
15530 // Ignore widening patterns.
15531 if (IdxVal == 0 && Vec.isUndef())
15532 return SDValue();
15533
15534 // Subvector must be half the width and an "aligned" insertion.
15535 unsigned NumSubElts = SubVT.getVectorNumElements();
15536 if ((SubVT.getSizeInBits() * 2) != VecVT.getSizeInBits() ||
15537 (IdxVal != 0 && IdxVal != NumSubElts))
15538 return SDValue();
15539
15540 // Fold insert_subvector -> concat_vectors
15541 // insert_subvector(Vec,Sub,lo) -> concat_vectors(Sub,extract(Vec,hi))
15542 // insert_subvector(Vec,Sub,hi) -> concat_vectors(extract(Vec,lo),Sub)
15543 SDLoc DL(N);
15544 SDValue Lo, Hi;
15545 if (IdxVal == 0) {
15546 Lo = SubVec;
15547 Hi = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec,
15548 DCI.DAG.getVectorIdxConstant(NumSubElts, DL));
15549 } else {
15550 Lo = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec,
15551 DCI.DAG.getVectorIdxConstant(0, DL));
15552 Hi = SubVec;
15553 }
15554 return DCI.DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, Hi);
15555 }
15556
15557 // shuffle(MVETrunc(x, y)) -> VMOVN(x, y)
PerformShuffleVMOVNCombine(ShuffleVectorSDNode * N,SelectionDAG & DAG)15558 static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N,
15559 SelectionDAG &DAG) {
15560 SDValue Trunc = N->getOperand(0);
15561 EVT VT = Trunc.getValueType();
15562 if (Trunc.getOpcode() != ARMISD::MVETRUNC || !N->getOperand(1).isUndef())
15563 return SDValue();
15564
15565 SDLoc DL(Trunc);
15566 if (isVMOVNTruncMask(N->getMask(), VT, false))
15567 return DAG.getNode(
15568 ARMISD::VMOVN, DL, VT,
15569 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)),
15570 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)),
15571 DAG.getConstant(1, DL, MVT::i32));
15572 else if (isVMOVNTruncMask(N->getMask(), VT, true))
15573 return DAG.getNode(
15574 ARMISD::VMOVN, DL, VT,
15575 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)),
15576 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)),
15577 DAG.getConstant(1, DL, MVT::i32));
15578 return SDValue();
15579 }
15580
15581 /// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
15582 /// ISD::VECTOR_SHUFFLE.
PerformVECTOR_SHUFFLECombine(SDNode * N,SelectionDAG & DAG)15583 static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
15584 if (SDValue R = FlattenVectorShuffle(cast<ShuffleVectorSDNode>(N), DAG))
15585 return R;
15586 if (SDValue R = PerformShuffleVMOVNCombine(cast<ShuffleVectorSDNode>(N), DAG))
15587 return R;
15588
15589 // The LLVM shufflevector instruction does not require the shuffle mask
15590 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
15591 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
15592 // operands do not match the mask length, they are extended by concatenating
15593 // them with undef vectors. That is probably the right thing for other
15594 // targets, but for NEON it is better to concatenate two double-register
15595 // size vector operands into a single quad-register size vector. Do that
15596 // transformation here:
15597 // shuffle(concat(v1, undef), concat(v2, undef)) ->
15598 // shuffle(concat(v1, v2), undef)
15599 SDValue Op0 = N->getOperand(0);
15600 SDValue Op1 = N->getOperand(1);
15601 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
15602 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
15603 Op0.getNumOperands() != 2 ||
15604 Op1.getNumOperands() != 2)
15605 return SDValue();
15606 SDValue Concat0Op1 = Op0.getOperand(1);
15607 SDValue Concat1Op1 = Op1.getOperand(1);
15608 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef())
15609 return SDValue();
15610 // Skip the transformation if any of the types are illegal.
15611 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15612 EVT VT = N->getValueType(0);
15613 if (!TLI.isTypeLegal(VT) ||
15614 !TLI.isTypeLegal(Concat0Op1.getValueType()) ||
15615 !TLI.isTypeLegal(Concat1Op1.getValueType()))
15616 return SDValue();
15617
15618 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
15619 Op0.getOperand(0), Op1.getOperand(0));
15620 // Translate the shuffle mask.
15621 SmallVector<int, 16> NewMask;
15622 unsigned NumElts = VT.getVectorNumElements();
15623 unsigned HalfElts = NumElts/2;
15624 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
15625 for (unsigned n = 0; n < NumElts; ++n) {
15626 int MaskElt = SVN->getMaskElt(n);
15627 int NewElt = -1;
15628 if (MaskElt < (int)HalfElts)
15629 NewElt = MaskElt;
15630 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
15631 NewElt = HalfElts + MaskElt - NumElts;
15632 NewMask.push_back(NewElt);
15633 }
15634 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat,
15635 DAG.getUNDEF(VT), NewMask);
15636 }
15637
15638 /// Load/store instruction that can be merged with a base address
15639 /// update
15640 struct BaseUpdateTarget {
15641 SDNode *N;
15642 bool isIntrinsic;
15643 bool isStore;
15644 unsigned AddrOpIdx;
15645 };
15646
15647 struct BaseUpdateUser {
15648 /// Instruction that updates a pointer
15649 SDNode *N;
15650 /// Pointer increment operand
15651 SDValue Inc;
15652 /// Pointer increment value if it is a constant, or 0 otherwise
15653 unsigned ConstInc;
15654 };
15655
TryCombineBaseUpdate(struct BaseUpdateTarget & Target,struct BaseUpdateUser & User,bool SimpleConstIncOnly,TargetLowering::DAGCombinerInfo & DCI)15656 static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target,
15657 struct BaseUpdateUser &User,
15658 bool SimpleConstIncOnly,
15659 TargetLowering::DAGCombinerInfo &DCI) {
15660 SelectionDAG &DAG = DCI.DAG;
15661 SDNode *N = Target.N;
15662 MemSDNode *MemN = cast<MemSDNode>(N);
15663 SDLoc dl(N);
15664
15665 // Find the new opcode for the updating load/store.
15666 bool isLoadOp = true;
15667 bool isLaneOp = false;
15668 // Workaround for vst1x and vld1x intrinsics which do not have alignment
15669 // as an operand.
15670 bool hasAlignment = true;
15671 unsigned NewOpc = 0;
15672 unsigned NumVecs = 0;
15673 if (Target.isIntrinsic) {
15674 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
15675 switch (IntNo) {
15676 default:
15677 llvm_unreachable("unexpected intrinsic for Neon base update");
15678 case Intrinsic::arm_neon_vld1:
15679 NewOpc = ARMISD::VLD1_UPD;
15680 NumVecs = 1;
15681 break;
15682 case Intrinsic::arm_neon_vld2:
15683 NewOpc = ARMISD::VLD2_UPD;
15684 NumVecs = 2;
15685 break;
15686 case Intrinsic::arm_neon_vld3:
15687 NewOpc = ARMISD::VLD3_UPD;
15688 NumVecs = 3;
15689 break;
15690 case Intrinsic::arm_neon_vld4:
15691 NewOpc = ARMISD::VLD4_UPD;
15692 NumVecs = 4;
15693 break;
15694 case Intrinsic::arm_neon_vld1x2:
15695 NewOpc = ARMISD::VLD1x2_UPD;
15696 NumVecs = 2;
15697 hasAlignment = false;
15698 break;
15699 case Intrinsic::arm_neon_vld1x3:
15700 NewOpc = ARMISD::VLD1x3_UPD;
15701 NumVecs = 3;
15702 hasAlignment = false;
15703 break;
15704 case Intrinsic::arm_neon_vld1x4:
15705 NewOpc = ARMISD::VLD1x4_UPD;
15706 NumVecs = 4;
15707 hasAlignment = false;
15708 break;
15709 case Intrinsic::arm_neon_vld2dup:
15710 NewOpc = ARMISD::VLD2DUP_UPD;
15711 NumVecs = 2;
15712 break;
15713 case Intrinsic::arm_neon_vld3dup:
15714 NewOpc = ARMISD::VLD3DUP_UPD;
15715 NumVecs = 3;
15716 break;
15717 case Intrinsic::arm_neon_vld4dup:
15718 NewOpc = ARMISD::VLD4DUP_UPD;
15719 NumVecs = 4;
15720 break;
15721 case Intrinsic::arm_neon_vld2lane:
15722 NewOpc = ARMISD::VLD2LN_UPD;
15723 NumVecs = 2;
15724 isLaneOp = true;
15725 break;
15726 case Intrinsic::arm_neon_vld3lane:
15727 NewOpc = ARMISD::VLD3LN_UPD;
15728 NumVecs = 3;
15729 isLaneOp = true;
15730 break;
15731 case Intrinsic::arm_neon_vld4lane:
15732 NewOpc = ARMISD::VLD4LN_UPD;
15733 NumVecs = 4;
15734 isLaneOp = true;
15735 break;
15736 case Intrinsic::arm_neon_vst1:
15737 NewOpc = ARMISD::VST1_UPD;
15738 NumVecs = 1;
15739 isLoadOp = false;
15740 break;
15741 case Intrinsic::arm_neon_vst2:
15742 NewOpc = ARMISD::VST2_UPD;
15743 NumVecs = 2;
15744 isLoadOp = false;
15745 break;
15746 case Intrinsic::arm_neon_vst3:
15747 NewOpc = ARMISD::VST3_UPD;
15748 NumVecs = 3;
15749 isLoadOp = false;
15750 break;
15751 case Intrinsic::arm_neon_vst4:
15752 NewOpc = ARMISD::VST4_UPD;
15753 NumVecs = 4;
15754 isLoadOp = false;
15755 break;
15756 case Intrinsic::arm_neon_vst2lane:
15757 NewOpc = ARMISD::VST2LN_UPD;
15758 NumVecs = 2;
15759 isLoadOp = false;
15760 isLaneOp = true;
15761 break;
15762 case Intrinsic::arm_neon_vst3lane:
15763 NewOpc = ARMISD::VST3LN_UPD;
15764 NumVecs = 3;
15765 isLoadOp = false;
15766 isLaneOp = true;
15767 break;
15768 case Intrinsic::arm_neon_vst4lane:
15769 NewOpc = ARMISD::VST4LN_UPD;
15770 NumVecs = 4;
15771 isLoadOp = false;
15772 isLaneOp = true;
15773 break;
15774 case Intrinsic::arm_neon_vst1x2:
15775 NewOpc = ARMISD::VST1x2_UPD;
15776 NumVecs = 2;
15777 isLoadOp = false;
15778 hasAlignment = false;
15779 break;
15780 case Intrinsic::arm_neon_vst1x3:
15781 NewOpc = ARMISD::VST1x3_UPD;
15782 NumVecs = 3;
15783 isLoadOp = false;
15784 hasAlignment = false;
15785 break;
15786 case Intrinsic::arm_neon_vst1x4:
15787 NewOpc = ARMISD::VST1x4_UPD;
15788 NumVecs = 4;
15789 isLoadOp = false;
15790 hasAlignment = false;
15791 break;
15792 }
15793 } else {
15794 isLaneOp = true;
15795 switch (N->getOpcode()) {
15796 default:
15797 llvm_unreachable("unexpected opcode for Neon base update");
15798 case ARMISD::VLD1DUP:
15799 NewOpc = ARMISD::VLD1DUP_UPD;
15800 NumVecs = 1;
15801 break;
15802 case ARMISD::VLD2DUP:
15803 NewOpc = ARMISD::VLD2DUP_UPD;
15804 NumVecs = 2;
15805 break;
15806 case ARMISD::VLD3DUP:
15807 NewOpc = ARMISD::VLD3DUP_UPD;
15808 NumVecs = 3;
15809 break;
15810 case ARMISD::VLD4DUP:
15811 NewOpc = ARMISD::VLD4DUP_UPD;
15812 NumVecs = 4;
15813 break;
15814 case ISD::LOAD:
15815 NewOpc = ARMISD::VLD1_UPD;
15816 NumVecs = 1;
15817 isLaneOp = false;
15818 break;
15819 case ISD::STORE:
15820 NewOpc = ARMISD::VST1_UPD;
15821 NumVecs = 1;
15822 isLaneOp = false;
15823 isLoadOp = false;
15824 break;
15825 }
15826 }
15827
15828 // Find the size of memory referenced by the load/store.
15829 EVT VecTy;
15830 if (isLoadOp) {
15831 VecTy = N->getValueType(0);
15832 } else if (Target.isIntrinsic) {
15833 VecTy = N->getOperand(Target.AddrOpIdx + 1).getValueType();
15834 } else {
15835 assert(Target.isStore &&
15836 "Node has to be a load, a store, or an intrinsic!");
15837 VecTy = N->getOperand(1).getValueType();
15838 }
15839
15840 bool isVLDDUPOp =
15841 NewOpc == ARMISD::VLD1DUP_UPD || NewOpc == ARMISD::VLD2DUP_UPD ||
15842 NewOpc == ARMISD::VLD3DUP_UPD || NewOpc == ARMISD::VLD4DUP_UPD;
15843
15844 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
15845 if (isLaneOp || isVLDDUPOp)
15846 NumBytes /= VecTy.getVectorNumElements();
15847
15848 if (NumBytes >= 3 * 16 && User.ConstInc != NumBytes) {
15849 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
15850 // separate instructions that make it harder to use a non-constant update.
15851 return false;
15852 }
15853
15854 if (SimpleConstIncOnly && User.ConstInc != NumBytes)
15855 return false;
15856
15857 // OK, we found an ADD we can fold into the base update.
15858 // Now, create a _UPD node, taking care of not breaking alignment.
15859
15860 EVT AlignedVecTy = VecTy;
15861 Align Alignment = MemN->getAlign();
15862
15863 // If this is a less-than-standard-aligned load/store, change the type to
15864 // match the standard alignment.
15865 // The alignment is overlooked when selecting _UPD variants; and it's
15866 // easier to introduce bitcasts here than fix that.
15867 // There are 3 ways to get to this base-update combine:
15868 // - intrinsics: they are assumed to be properly aligned (to the standard
15869 // alignment of the memory type), so we don't need to do anything.
15870 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
15871 // intrinsics, so, likewise, there's nothing to do.
15872 // - generic load/store instructions: the alignment is specified as an
15873 // explicit operand, rather than implicitly as the standard alignment
15874 // of the memory type (like the intrisics). We need to change the
15875 // memory type to match the explicit alignment. That way, we don't
15876 // generate non-standard-aligned ARMISD::VLDx nodes.
15877 if (isa<LSBaseSDNode>(N)) {
15878 if (Alignment.value() < VecTy.getScalarSizeInBits() / 8) {
15879 MVT EltTy = MVT::getIntegerVT(Alignment.value() * 8);
15880 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.");
15881 assert(!isLaneOp && "Unexpected generic load/store lane.");
15882 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
15883 AlignedVecTy = MVT::getVectorVT(EltTy, NumElts);
15884 }
15885 // Don't set an explicit alignment on regular load/stores that we want
15886 // to transform to VLD/VST 1_UPD nodes.
15887 // This matches the behavior of regular load/stores, which only get an
15888 // explicit alignment if the MMO alignment is larger than the standard
15889 // alignment of the memory type.
15890 // Intrinsics, however, always get an explicit alignment, set to the
15891 // alignment of the MMO.
15892 Alignment = Align(1);
15893 }
15894
15895 // Create the new updating load/store node.
15896 // First, create an SDVTList for the new updating node's results.
15897 EVT Tys[6];
15898 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
15899 unsigned n;
15900 for (n = 0; n < NumResultVecs; ++n)
15901 Tys[n] = AlignedVecTy;
15902 Tys[n++] = MVT::i32;
15903 Tys[n] = MVT::Other;
15904 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumResultVecs + 2));
15905
15906 // Then, gather the new node's operands.
15907 SmallVector<SDValue, 8> Ops;
15908 Ops.push_back(N->getOperand(0)); // incoming chain
15909 Ops.push_back(N->getOperand(Target.AddrOpIdx));
15910 Ops.push_back(User.Inc);
15911
15912 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(N)) {
15913 // Try to match the intrinsic's signature
15914 Ops.push_back(StN->getValue());
15915 } else {
15916 // Loads (and of course intrinsics) match the intrinsics' signature,
15917 // so just add all but the alignment operand.
15918 unsigned LastOperand =
15919 hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands();
15920 for (unsigned i = Target.AddrOpIdx + 1; i < LastOperand; ++i)
15921 Ops.push_back(N->getOperand(i));
15922 }
15923
15924 // For all node types, the alignment operand is always the last one.
15925 Ops.push_back(DAG.getConstant(Alignment.value(), dl, MVT::i32));
15926
15927 // If this is a non-standard-aligned STORE, the penultimate operand is the
15928 // stored value. Bitcast it to the aligned type.
15929 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
15930 SDValue &StVal = Ops[Ops.size() - 2];
15931 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
15932 }
15933
15934 EVT LoadVT = isLaneOp ? VecTy.getVectorElementType() : AlignedVecTy;
15935 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT,
15936 MemN->getMemOperand());
15937
15938 // Update the uses.
15939 SmallVector<SDValue, 5> NewResults;
15940 for (unsigned i = 0; i < NumResultVecs; ++i)
15941 NewResults.push_back(SDValue(UpdN.getNode(), i));
15942
15943 // If this is an non-standard-aligned LOAD, the first result is the loaded
15944 // value. Bitcast it to the expected result type.
15945 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
15946 SDValue &LdVal = NewResults[0];
15947 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal);
15948 }
15949
15950 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
15951 DCI.CombineTo(N, NewResults);
15952 DCI.CombineTo(User.N, SDValue(UpdN.getNode(), NumResultVecs));
15953
15954 return true;
15955 }
15956
15957 // If (opcode ptr inc) is and ADD-like instruction, return the
15958 // increment value. Otherwise return 0.
getPointerConstIncrement(unsigned Opcode,SDValue Ptr,SDValue Inc,const SelectionDAG & DAG)15959 static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr,
15960 SDValue Inc, const SelectionDAG &DAG) {
15961 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode());
15962 if (!CInc)
15963 return 0;
15964
15965 switch (Opcode) {
15966 case ARMISD::VLD1_UPD:
15967 case ISD::ADD:
15968 return CInc->getZExtValue();
15969 case ISD::OR: {
15970 if (DAG.haveNoCommonBitsSet(Ptr, Inc)) {
15971 // (OR ptr inc) is the same as (ADD ptr inc)
15972 return CInc->getZExtValue();
15973 }
15974 return 0;
15975 }
15976 default:
15977 return 0;
15978 }
15979 }
15980
findPointerConstIncrement(SDNode * N,SDValue * Ptr,SDValue * CInc)15981 static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc) {
15982 switch (N->getOpcode()) {
15983 case ISD::ADD:
15984 case ISD::OR: {
15985 if (isa<ConstantSDNode>(N->getOperand(1))) {
15986 *Ptr = N->getOperand(0);
15987 *CInc = N->getOperand(1);
15988 return true;
15989 }
15990 return false;
15991 }
15992 case ARMISD::VLD1_UPD: {
15993 if (isa<ConstantSDNode>(N->getOperand(2))) {
15994 *Ptr = N->getOperand(1);
15995 *CInc = N->getOperand(2);
15996 return true;
15997 }
15998 return false;
15999 }
16000 default:
16001 return false;
16002 }
16003 }
16004
isValidBaseUpdate(SDNode * N,SDNode * User)16005 static bool isValidBaseUpdate(SDNode *N, SDNode *User) {
16006 // Check that the add is independent of the load/store.
16007 // Otherwise, folding it would create a cycle. Search through Addr
16008 // as well, since the User may not be a direct user of Addr and
16009 // only share a base pointer.
16010 SmallPtrSet<const SDNode *, 32> Visited;
16011 SmallVector<const SDNode *, 16> Worklist;
16012 Worklist.push_back(N);
16013 Worklist.push_back(User);
16014 if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
16015 SDNode::hasPredecessorHelper(User, Visited, Worklist))
16016 return false;
16017 return true;
16018 }
16019
16020 /// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
16021 /// NEON load/store intrinsics, and generic vector load/stores, to merge
16022 /// base address updates.
16023 /// For generic load/stores, the memory type is assumed to be a vector.
16024 /// The caller is assumed to have checked legality.
CombineBaseUpdate(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)16025 static SDValue CombineBaseUpdate(SDNode *N,
16026 TargetLowering::DAGCombinerInfo &DCI) {
16027 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
16028 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
16029 const bool isStore = N->getOpcode() == ISD::STORE;
16030 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
16031 BaseUpdateTarget Target = {N, isIntrinsic, isStore, AddrOpIdx};
16032
16033 SDValue Addr = N->getOperand(AddrOpIdx);
16034
16035 SmallVector<BaseUpdateUser, 8> BaseUpdates;
16036
16037 // Search for a use of the address operand that is an increment.
16038 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
16039 UE = Addr.getNode()->use_end(); UI != UE; ++UI) {
16040 SDNode *User = *UI;
16041 if (UI.getUse().getResNo() != Addr.getResNo() ||
16042 User->getNumOperands() != 2)
16043 continue;
16044
16045 SDValue Inc = User->getOperand(UI.getOperandNo() == 1 ? 0 : 1);
16046 unsigned ConstInc =
16047 getPointerConstIncrement(User->getOpcode(), Addr, Inc, DCI.DAG);
16048
16049 if (ConstInc || User->getOpcode() == ISD::ADD)
16050 BaseUpdates.push_back({User, Inc, ConstInc});
16051 }
16052
16053 // If the address is a constant pointer increment itself, find
16054 // another constant increment that has the same base operand
16055 SDValue Base;
16056 SDValue CInc;
16057 if (findPointerConstIncrement(Addr.getNode(), &Base, &CInc)) {
16058 unsigned Offset =
16059 getPointerConstIncrement(Addr->getOpcode(), Base, CInc, DCI.DAG);
16060 for (SDNode::use_iterator UI = Base->use_begin(), UE = Base->use_end();
16061 UI != UE; ++UI) {
16062
16063 SDNode *User = *UI;
16064 if (UI.getUse().getResNo() != Base.getResNo() || User == Addr.getNode() ||
16065 User->getNumOperands() != 2)
16066 continue;
16067
16068 SDValue UserInc = User->getOperand(UI.getOperandNo() == 0 ? 1 : 0);
16069 unsigned UserOffset =
16070 getPointerConstIncrement(User->getOpcode(), Base, UserInc, DCI.DAG);
16071
16072 if (!UserOffset || UserOffset <= Offset)
16073 continue;
16074
16075 unsigned NewConstInc = UserOffset - Offset;
16076 SDValue NewInc = DCI.DAG.getConstant(NewConstInc, SDLoc(N), MVT::i32);
16077 BaseUpdates.push_back({User, NewInc, NewConstInc});
16078 }
16079 }
16080
16081 // Try to fold the load/store with an update that matches memory
16082 // access size. This should work well for sequential loads.
16083 //
16084 // Filter out invalid updates as well.
16085 unsigned NumValidUpd = BaseUpdates.size();
16086 for (unsigned I = 0; I < NumValidUpd;) {
16087 BaseUpdateUser &User = BaseUpdates[I];
16088 if (!isValidBaseUpdate(N, User.N)) {
16089 --NumValidUpd;
16090 std::swap(BaseUpdates[I], BaseUpdates[NumValidUpd]);
16091 continue;
16092 }
16093
16094 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/true, DCI))
16095 return SDValue();
16096 ++I;
16097 }
16098 BaseUpdates.resize(NumValidUpd);
16099
16100 // Try to fold with other users. Non-constant updates are considered
16101 // first, and constant updates are sorted to not break a sequence of
16102 // strided accesses (if there is any).
16103 std::stable_sort(BaseUpdates.begin(), BaseUpdates.end(),
16104 [](const BaseUpdateUser &LHS, const BaseUpdateUser &RHS) {
16105 return LHS.ConstInc < RHS.ConstInc;
16106 });
16107 for (BaseUpdateUser &User : BaseUpdates) {
16108 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/false, DCI))
16109 return SDValue();
16110 }
16111 return SDValue();
16112 }
16113
PerformVLDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)16114 static SDValue PerformVLDCombine(SDNode *N,
16115 TargetLowering::DAGCombinerInfo &DCI) {
16116 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16117 return SDValue();
16118
16119 return CombineBaseUpdate(N, DCI);
16120 }
16121
PerformMVEVLDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)16122 static SDValue PerformMVEVLDCombine(SDNode *N,
16123 TargetLowering::DAGCombinerInfo &DCI) {
16124 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16125 return SDValue();
16126
16127 SelectionDAG &DAG = DCI.DAG;
16128 SDValue Addr = N->getOperand(2);
16129 MemSDNode *MemN = cast<MemSDNode>(N);
16130 SDLoc dl(N);
16131
16132 // For the stores, where there are multiple intrinsics we only actually want
16133 // to post-inc the last of the them.
16134 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
16135 if (IntNo == Intrinsic::arm_mve_vst2q &&
16136 cast<ConstantSDNode>(N->getOperand(5))->getZExtValue() != 1)
16137 return SDValue();
16138 if (IntNo == Intrinsic::arm_mve_vst4q &&
16139 cast<ConstantSDNode>(N->getOperand(7))->getZExtValue() != 3)
16140 return SDValue();
16141
16142 // Search for a use of the address operand that is an increment.
16143 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
16144 UE = Addr.getNode()->use_end();
16145 UI != UE; ++UI) {
16146 SDNode *User = *UI;
16147 if (User->getOpcode() != ISD::ADD ||
16148 UI.getUse().getResNo() != Addr.getResNo())
16149 continue;
16150
16151 // Check that the add is independent of the load/store. Otherwise, folding
16152 // it would create a cycle. We can avoid searching through Addr as it's a
16153 // predecessor to both.
16154 SmallPtrSet<const SDNode *, 32> Visited;
16155 SmallVector<const SDNode *, 16> Worklist;
16156 Visited.insert(Addr.getNode());
16157 Worklist.push_back(N);
16158 Worklist.push_back(User);
16159 if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
16160 SDNode::hasPredecessorHelper(User, Visited, Worklist))
16161 continue;
16162
16163 // Find the new opcode for the updating load/store.
16164 bool isLoadOp = true;
16165 unsigned NewOpc = 0;
16166 unsigned NumVecs = 0;
16167 switch (IntNo) {
16168 default:
16169 llvm_unreachable("unexpected intrinsic for MVE VLDn combine");
16170 case Intrinsic::arm_mve_vld2q:
16171 NewOpc = ARMISD::VLD2_UPD;
16172 NumVecs = 2;
16173 break;
16174 case Intrinsic::arm_mve_vld4q:
16175 NewOpc = ARMISD::VLD4_UPD;
16176 NumVecs = 4;
16177 break;
16178 case Intrinsic::arm_mve_vst2q:
16179 NewOpc = ARMISD::VST2_UPD;
16180 NumVecs = 2;
16181 isLoadOp = false;
16182 break;
16183 case Intrinsic::arm_mve_vst4q:
16184 NewOpc = ARMISD::VST4_UPD;
16185 NumVecs = 4;
16186 isLoadOp = false;
16187 break;
16188 }
16189
16190 // Find the size of memory referenced by the load/store.
16191 EVT VecTy;
16192 if (isLoadOp) {
16193 VecTy = N->getValueType(0);
16194 } else {
16195 VecTy = N->getOperand(3).getValueType();
16196 }
16197
16198 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16199
16200 // If the increment is a constant, it must match the memory ref size.
16201 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
16202 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode());
16203 if (!CInc || CInc->getZExtValue() != NumBytes)
16204 continue;
16205
16206 // Create the new updating load/store node.
16207 // First, create an SDVTList for the new updating node's results.
16208 EVT Tys[6];
16209 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16210 unsigned n;
16211 for (n = 0; n < NumResultVecs; ++n)
16212 Tys[n] = VecTy;
16213 Tys[n++] = MVT::i32;
16214 Tys[n] = MVT::Other;
16215 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumResultVecs + 2));
16216
16217 // Then, gather the new node's operands.
16218 SmallVector<SDValue, 8> Ops;
16219 Ops.push_back(N->getOperand(0)); // incoming chain
16220 Ops.push_back(N->getOperand(2)); // ptr
16221 Ops.push_back(Inc);
16222
16223 for (unsigned i = 3; i < N->getNumOperands(); ++i)
16224 Ops.push_back(N->getOperand(i));
16225
16226 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, VecTy,
16227 MemN->getMemOperand());
16228
16229 // Update the uses.
16230 SmallVector<SDValue, 5> NewResults;
16231 for (unsigned i = 0; i < NumResultVecs; ++i)
16232 NewResults.push_back(SDValue(UpdN.getNode(), i));
16233
16234 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16235 DCI.CombineTo(N, NewResults);
16236 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
16237
16238 break;
16239 }
16240
16241 return SDValue();
16242 }
16243
16244 /// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
16245 /// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
16246 /// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
16247 /// return true.
CombineVLDDUP(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)16248 static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
16249 SelectionDAG &DAG = DCI.DAG;
16250 EVT VT = N->getValueType(0);
16251 // vldN-dup instructions only support 64-bit vectors for N > 1.
16252 if (!VT.is64BitVector())
16253 return false;
16254
16255 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
16256 SDNode *VLD = N->getOperand(0).getNode();
16257 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
16258 return false;
16259 unsigned NumVecs = 0;
16260 unsigned NewOpc = 0;
16261 unsigned IntNo = cast<ConstantSDNode>(VLD->getOperand(1))->getZExtValue();
16262 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16263 NumVecs = 2;
16264 NewOpc = ARMISD::VLD2DUP;
16265 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16266 NumVecs = 3;
16267 NewOpc = ARMISD::VLD3DUP;
16268 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16269 NumVecs = 4;
16270 NewOpc = ARMISD::VLD4DUP;
16271 } else {
16272 return false;
16273 }
16274
16275 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
16276 // numbers match the load.
16277 unsigned VLDLaneNo =
16278 cast<ConstantSDNode>(VLD->getOperand(NumVecs+3))->getZExtValue();
16279 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
16280 UI != UE; ++UI) {
16281 // Ignore uses of the chain result.
16282 if (UI.getUse().getResNo() == NumVecs)
16283 continue;
16284 SDNode *User = *UI;
16285 if (User->getOpcode() != ARMISD::VDUPLANE ||
16286 VLDLaneNo != cast<ConstantSDNode>(User->getOperand(1))->getZExtValue())
16287 return false;
16288 }
16289
16290 // Create the vldN-dup node.
16291 EVT Tys[5];
16292 unsigned n;
16293 for (n = 0; n < NumVecs; ++n)
16294 Tys[n] = VT;
16295 Tys[n] = MVT::Other;
16296 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumVecs + 1));
16297 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) };
16298 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(VLD);
16299 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
16300 Ops, VLDMemInt->getMemoryVT(),
16301 VLDMemInt->getMemOperand());
16302
16303 // Update the uses.
16304 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
16305 UI != UE; ++UI) {
16306 unsigned ResNo = UI.getUse().getResNo();
16307 // Ignore uses of the chain result.
16308 if (ResNo == NumVecs)
16309 continue;
16310 SDNode *User = *UI;
16311 DCI.CombineTo(User, SDValue(VLDDup.getNode(), ResNo));
16312 }
16313
16314 // Now the vldN-lane intrinsic is dead except for its chain result.
16315 // Update uses of the chain.
16316 std::vector<SDValue> VLDDupResults;
16317 for (unsigned n = 0; n < NumVecs; ++n)
16318 VLDDupResults.push_back(SDValue(VLDDup.getNode(), n));
16319 VLDDupResults.push_back(SDValue(VLDDup.getNode(), NumVecs));
16320 DCI.CombineTo(VLD, VLDDupResults);
16321
16322 return true;
16323 }
16324
16325 /// PerformVDUPLANECombine - Target-specific dag combine xforms for
16326 /// ARMISD::VDUPLANE.
PerformVDUPLANECombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)16327 static SDValue PerformVDUPLANECombine(SDNode *N,
16328 TargetLowering::DAGCombinerInfo &DCI,
16329 const ARMSubtarget *Subtarget) {
16330 SDValue Op = N->getOperand(0);
16331 EVT VT = N->getValueType(0);
16332
16333 // On MVE, we just convert the VDUPLANE to a VDUP with an extract.
16334 if (Subtarget->hasMVEIntegerOps()) {
16335 EVT ExtractVT = VT.getVectorElementType();
16336 // We need to ensure we are creating a legal type.
16337 if (!DCI.DAG.getTargetLoweringInfo().isTypeLegal(ExtractVT))
16338 ExtractVT = MVT::i32;
16339 SDValue Extract = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), ExtractVT,
16340 N->getOperand(0), N->getOperand(1));
16341 return DCI.DAG.getNode(ARMISD::VDUP, SDLoc(N), VT, Extract);
16342 }
16343
16344 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
16345 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
16346 if (CombineVLDDUP(N, DCI))
16347 return SDValue(N, 0);
16348
16349 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
16350 // redundant. Ignore bit_converts for now; element sizes are checked below.
16351 while (Op.getOpcode() == ISD::BITCAST)
16352 Op = Op.getOperand(0);
16353 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
16354 return SDValue();
16355
16356 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
16357 unsigned EltSize = Op.getScalarValueSizeInBits();
16358 // The canonical VMOV for a zero vector uses a 32-bit element size.
16359 unsigned Imm = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
16360 unsigned EltBits;
16361 if (ARM_AM::decodeVMOVModImm(Imm, EltBits) == 0)
16362 EltSize = 8;
16363 if (EltSize > VT.getScalarSizeInBits())
16364 return SDValue();
16365
16366 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
16367 }
16368
16369 /// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
PerformVDUPCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)16370 static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG,
16371 const ARMSubtarget *Subtarget) {
16372 SDValue Op = N->getOperand(0);
16373 SDLoc dl(N);
16374
16375 if (Subtarget->hasMVEIntegerOps()) {
16376 // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will
16377 // need to come from a GPR.
16378 if (Op.getValueType() == MVT::f32)
16379 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
16380 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op));
16381 else if (Op.getValueType() == MVT::f16)
16382 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
16383 DAG.getNode(ARMISD::VMOVrh, dl, MVT::i32, Op));
16384 }
16385
16386 if (!Subtarget->hasNEON())
16387 return SDValue();
16388
16389 // Match VDUP(LOAD) -> VLD1DUP.
16390 // We match this pattern here rather than waiting for isel because the
16391 // transform is only legal for unindexed loads.
16392 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode());
16393 if (LD && Op.hasOneUse() && LD->isUnindexed() &&
16394 LD->getMemoryVT() == N->getValueType(0).getVectorElementType()) {
16395 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16396 DAG.getConstant(LD->getAlign().value(), SDLoc(N), MVT::i32)};
16397 SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other);
16398 SDValue VLDDup =
16399 DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys, Ops,
16400 LD->getMemoryVT(), LD->getMemOperand());
16401 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1));
16402 return VLDDup;
16403 }
16404
16405 return SDValue();
16406 }
16407
PerformLOADCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)16408 static SDValue PerformLOADCombine(SDNode *N,
16409 TargetLowering::DAGCombinerInfo &DCI,
16410 const ARMSubtarget *Subtarget) {
16411 EVT VT = N->getValueType(0);
16412
16413 // If this is a legal vector load, try to combine it into a VLD1_UPD.
16414 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() &&
16415 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16416 return CombineBaseUpdate(N, DCI);
16417
16418 return SDValue();
16419 }
16420
16421 // Optimize trunc store (of multiple scalars) to shuffle and store. First,
16422 // pack all of the elements in one place. Next, store to memory in fewer
16423 // chunks.
PerformTruncatingStoreCombine(StoreSDNode * St,SelectionDAG & DAG)16424 static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
16425 SelectionDAG &DAG) {
16426 SDValue StVal = St->getValue();
16427 EVT VT = StVal.getValueType();
16428 if (!St->isTruncatingStore() || !VT.isVector())
16429 return SDValue();
16430 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16431 EVT StVT = St->getMemoryVT();
16432 unsigned NumElems = VT.getVectorNumElements();
16433 assert(StVT != VT && "Cannot truncate to the same type");
16434 unsigned FromEltSz = VT.getScalarSizeInBits();
16435 unsigned ToEltSz = StVT.getScalarSizeInBits();
16436
16437 // From, To sizes and ElemCount must be pow of two
16438 if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz))
16439 return SDValue();
16440
16441 // We are going to use the original vector elt for storing.
16442 // Accumulated smaller vector elements must be a multiple of the store size.
16443 if (0 != (NumElems * FromEltSz) % ToEltSz)
16444 return SDValue();
16445
16446 unsigned SizeRatio = FromEltSz / ToEltSz;
16447 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
16448
16449 // Create a type on which we perform the shuffle.
16450 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
16451 NumElems * SizeRatio);
16452 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
16453
16454 SDLoc DL(St);
16455 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
16456 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
16457 for (unsigned i = 0; i < NumElems; ++i)
16458 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
16459 : i * SizeRatio;
16460
16461 // Can't shuffle using an illegal type.
16462 if (!TLI.isTypeLegal(WideVecVT))
16463 return SDValue();
16464
16465 SDValue Shuff = DAG.getVectorShuffle(
16466 WideVecVT, DL, WideVec, DAG.getUNDEF(WideVec.getValueType()), ShuffleVec);
16467 // At this point all of the data is stored at the bottom of the
16468 // register. We now need to save it to mem.
16469
16470 // Find the largest store unit
16471 MVT StoreType = MVT::i8;
16472 for (MVT Tp : MVT::integer_valuetypes()) {
16473 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16474 StoreType = Tp;
16475 }
16476 // Didn't find a legal store type.
16477 if (!TLI.isTypeLegal(StoreType))
16478 return SDValue();
16479
16480 // Bitcast the original vector into a vector of store-size units
16481 EVT StoreVecVT =
16482 EVT::getVectorVT(*DAG.getContext(), StoreType,
16483 VT.getSizeInBits() / EVT(StoreType).getSizeInBits());
16484 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
16485 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
16486 SmallVector<SDValue, 8> Chains;
16487 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
16488 TLI.getPointerTy(DAG.getDataLayout()));
16489 SDValue BasePtr = St->getBasePtr();
16490
16491 // Perform one or more big stores into memory.
16492 unsigned E = (ToEltSz * NumElems) / StoreType.getSizeInBits();
16493 for (unsigned I = 0; I < E; I++) {
16494 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType,
16495 ShuffWide, DAG.getIntPtrConstant(I, DL));
16496 SDValue Ch =
16497 DAG.getStore(St->getChain(), DL, SubVec, BasePtr, St->getPointerInfo(),
16498 St->getAlign(), St->getMemOperand()->getFlags());
16499 BasePtr =
16500 DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, Increment);
16501 Chains.push_back(Ch);
16502 }
16503 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
16504 }
16505
16506 // Try taking a single vector store from an fpround (which would otherwise turn
16507 // into an expensive buildvector) and splitting it into a series of narrowing
16508 // stores.
PerformSplittingToNarrowingStores(StoreSDNode * St,SelectionDAG & DAG)16509 static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
16510 SelectionDAG &DAG) {
16511 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16512 return SDValue();
16513 SDValue Trunc = St->getValue();
16514 if (Trunc->getOpcode() != ISD::FP_ROUND)
16515 return SDValue();
16516 EVT FromVT = Trunc->getOperand(0).getValueType();
16517 EVT ToVT = Trunc.getValueType();
16518 if (!ToVT.isVector())
16519 return SDValue();
16520 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
16521 EVT ToEltVT = ToVT.getVectorElementType();
16522 EVT FromEltVT = FromVT.getVectorElementType();
16523
16524 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16525 return SDValue();
16526
16527 unsigned NumElements = 4;
16528 if (FromVT.getVectorNumElements() % NumElements != 0)
16529 return SDValue();
16530
16531 // Test if the Trunc will be convertable to a VMOVN with a shuffle, and if so
16532 // use the VMOVN over splitting the store. We are looking for patterns of:
16533 // !rev: 0 N 1 N+1 2 N+2 ...
16534 // rev: N 0 N+1 1 N+2 2 ...
16535 // The shuffle may either be a single source (in which case N = NumElts/2) or
16536 // two inputs extended with concat to the same size (in which case N =
16537 // NumElts).
16538 auto isVMOVNShuffle = [&](ShuffleVectorSDNode *SVN, bool Rev) {
16539 ArrayRef<int> M = SVN->getMask();
16540 unsigned NumElts = ToVT.getVectorNumElements();
16541 if (SVN->getOperand(1).isUndef())
16542 NumElts /= 2;
16543
16544 unsigned Off0 = Rev ? NumElts : 0;
16545 unsigned Off1 = Rev ? 0 : NumElts;
16546
16547 for (unsigned I = 0; I < NumElts; I += 2) {
16548 if (M[I] >= 0 && M[I] != (int)(Off0 + I / 2))
16549 return false;
16550 if (M[I + 1] >= 0 && M[I + 1] != (int)(Off1 + I / 2))
16551 return false;
16552 }
16553
16554 return true;
16555 };
16556
16557 if (auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.getOperand(0)))
16558 if (isVMOVNShuffle(Shuffle, false) || isVMOVNShuffle(Shuffle, true))
16559 return SDValue();
16560
16561 LLVMContext &C = *DAG.getContext();
16562 SDLoc DL(St);
16563 // Details about the old store
16564 SDValue Ch = St->getChain();
16565 SDValue BasePtr = St->getBasePtr();
16566 Align Alignment = St->getOriginalAlign();
16567 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16568 AAMDNodes AAInfo = St->getAAInfo();
16569
16570 // We split the store into slices of NumElements. fp16 trunc stores are vcvt
16571 // and then stored as truncating integer stores.
16572 EVT NewFromVT = EVT::getVectorVT(C, FromEltVT, NumElements);
16573 EVT NewToVT = EVT::getVectorVT(
16574 C, EVT::getIntegerVT(C, ToEltVT.getSizeInBits()), NumElements);
16575
16576 SmallVector<SDValue, 4> Stores;
16577 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
16578 unsigned NewOffset = i * NumElements * ToEltVT.getSizeInBits() / 8;
16579 SDValue NewPtr =
16580 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset));
16581
16582 SDValue Extract =
16583 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0),
16584 DAG.getConstant(i * NumElements, DL, MVT::i32));
16585
16586 SDValue FPTrunc =
16587 DAG.getNode(ARMISD::VCVTN, DL, MVT::v8f16, DAG.getUNDEF(MVT::v8f16),
16588 Extract, DAG.getConstant(0, DL, MVT::i32));
16589 Extract = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v4i32, FPTrunc);
16590
16591 SDValue Store = DAG.getTruncStore(
16592 Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
16593 NewToVT, Alignment, MMOFlags, AAInfo);
16594 Stores.push_back(Store);
16595 }
16596 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
16597 }
16598
16599 // Try taking a single vector store from an MVETRUNC (which would otherwise turn
16600 // into an expensive buildvector) and splitting it into a series of narrowing
16601 // stores.
PerformSplittingMVETruncToNarrowingStores(StoreSDNode * St,SelectionDAG & DAG)16602 static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St,
16603 SelectionDAG &DAG) {
16604 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16605 return SDValue();
16606 SDValue Trunc = St->getValue();
16607 if (Trunc->getOpcode() != ARMISD::MVETRUNC)
16608 return SDValue();
16609 EVT FromVT = Trunc->getOperand(0).getValueType();
16610 EVT ToVT = Trunc.getValueType();
16611
16612 LLVMContext &C = *DAG.getContext();
16613 SDLoc DL(St);
16614 // Details about the old store
16615 SDValue Ch = St->getChain();
16616 SDValue BasePtr = St->getBasePtr();
16617 Align Alignment = St->getOriginalAlign();
16618 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16619 AAMDNodes AAInfo = St->getAAInfo();
16620
16621 EVT NewToVT = EVT::getVectorVT(C, ToVT.getVectorElementType(),
16622 FromVT.getVectorNumElements());
16623
16624 SmallVector<SDValue, 4> Stores;
16625 for (unsigned i = 0; i < Trunc.getNumOperands(); i++) {
16626 unsigned NewOffset =
16627 i * FromVT.getVectorNumElements() * ToVT.getScalarSizeInBits() / 8;
16628 SDValue NewPtr =
16629 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset));
16630
16631 SDValue Extract = Trunc.getOperand(i);
16632 SDValue Store = DAG.getTruncStore(
16633 Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
16634 NewToVT, Alignment, MMOFlags, AAInfo);
16635 Stores.push_back(Store);
16636 }
16637 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
16638 }
16639
16640 // Given a floating point store from an extracted vector, with an integer
16641 // VGETLANE that already exists, store the existing VGETLANEu directly. This can
16642 // help reduce fp register pressure, doesn't require the fp extract and allows
16643 // use of more integer post-inc stores not available with vstr.
PerformExtractFpToIntStores(StoreSDNode * St,SelectionDAG & DAG)16644 static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG) {
16645 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16646 return SDValue();
16647 SDValue Extract = St->getValue();
16648 EVT VT = Extract.getValueType();
16649 // For now only uses f16. This may be useful for f32 too, but that will
16650 // be bitcast(extract), not the VGETLANEu we currently check here.
16651 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16652 return SDValue();
16653
16654 SDNode *GetLane =
16655 DAG.getNodeIfExists(ARMISD::VGETLANEu, DAG.getVTList(MVT::i32),
16656 {Extract.getOperand(0), Extract.getOperand(1)});
16657 if (!GetLane)
16658 return SDValue();
16659
16660 LLVMContext &C = *DAG.getContext();
16661 SDLoc DL(St);
16662 // Create a new integer store to replace the existing floating point version.
16663 SDValue Ch = St->getChain();
16664 SDValue BasePtr = St->getBasePtr();
16665 Align Alignment = St->getOriginalAlign();
16666 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16667 AAMDNodes AAInfo = St->getAAInfo();
16668 EVT NewToVT = EVT::getIntegerVT(C, VT.getSizeInBits());
16669 SDValue Store = DAG.getTruncStore(Ch, DL, SDValue(GetLane, 0), BasePtr,
16670 St->getPointerInfo(), NewToVT, Alignment,
16671 MMOFlags, AAInfo);
16672
16673 return Store;
16674 }
16675
16676 /// PerformSTORECombine - Target-specific dag combine xforms for
16677 /// ISD::STORE.
PerformSTORECombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)16678 static SDValue PerformSTORECombine(SDNode *N,
16679 TargetLowering::DAGCombinerInfo &DCI,
16680 const ARMSubtarget *Subtarget) {
16681 StoreSDNode *St = cast<StoreSDNode>(N);
16682 if (St->isVolatile())
16683 return SDValue();
16684 SDValue StVal = St->getValue();
16685 EVT VT = StVal.getValueType();
16686
16687 if (Subtarget->hasNEON())
16688 if (SDValue Store = PerformTruncatingStoreCombine(St, DCI.DAG))
16689 return Store;
16690
16691 if (Subtarget->hasMVEIntegerOps()) {
16692 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG))
16693 return NewToken;
16694 if (SDValue NewChain = PerformExtractFpToIntStores(St, DCI.DAG))
16695 return NewChain;
16696 if (SDValue NewToken =
16697 PerformSplittingMVETruncToNarrowingStores(St, DCI.DAG))
16698 return NewToken;
16699 }
16700
16701 if (!ISD::isNormalStore(St))
16702 return SDValue();
16703
16704 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
16705 // ARM stores of arguments in the same cache line.
16706 if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR &&
16707 StVal.getNode()->hasOneUse()) {
16708 SelectionDAG &DAG = DCI.DAG;
16709 bool isBigEndian = DAG.getDataLayout().isBigEndian();
16710 SDLoc DL(St);
16711 SDValue BasePtr = St->getBasePtr();
16712 SDValue NewST1 = DAG.getStore(
16713 St->getChain(), DL, StVal.getNode()->getOperand(isBigEndian ? 1 : 0),
16714 BasePtr, St->getPointerInfo(), St->getOriginalAlign(),
16715 St->getMemOperand()->getFlags());
16716
16717 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
16718 DAG.getConstant(4, DL, MVT::i32));
16719 return DAG.getStore(NewST1.getValue(0), DL,
16720 StVal.getNode()->getOperand(isBigEndian ? 0 : 1),
16721 OffsetPtr, St->getPointerInfo().getWithOffset(4),
16722 St->getOriginalAlign(),
16723 St->getMemOperand()->getFlags());
16724 }
16725
16726 if (StVal.getValueType() == MVT::i64 &&
16727 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
16728
16729 // Bitcast an i64 store extracted from a vector to f64.
16730 // Otherwise, the i64 value will be legalized to a pair of i32 values.
16731 SelectionDAG &DAG = DCI.DAG;
16732 SDLoc dl(StVal);
16733 SDValue IntVec = StVal.getOperand(0);
16734 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
16735 IntVec.getValueType().getVectorNumElements());
16736 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
16737 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
16738 Vec, StVal.getOperand(1));
16739 dl = SDLoc(N);
16740 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
16741 // Make the DAGCombiner fold the bitcasts.
16742 DCI.AddToWorklist(Vec.getNode());
16743 DCI.AddToWorklist(ExtElt.getNode());
16744 DCI.AddToWorklist(V.getNode());
16745 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(),
16746 St->getPointerInfo(), St->getAlign(),
16747 St->getMemOperand()->getFlags(), St->getAAInfo());
16748 }
16749
16750 // If this is a legal vector store, try to combine it into a VST1_UPD.
16751 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() &&
16752 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16753 return CombineBaseUpdate(N, DCI);
16754
16755 return SDValue();
16756 }
16757
16758 /// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
16759 /// can replace combinations of VMUL and VCVT (floating-point to integer)
16760 /// when the VMUL has a constant operand that is a power of 2.
16761 ///
16762 /// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
16763 /// vmul.f32 d16, d17, d16
16764 /// vcvt.s32.f32 d16, d16
16765 /// becomes:
16766 /// vcvt.s32.f32 d16, d16, #3
PerformVCVTCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)16767 static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
16768 const ARMSubtarget *Subtarget) {
16769 if (!Subtarget->hasNEON())
16770 return SDValue();
16771
16772 SDValue Op = N->getOperand(0);
16773 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
16774 Op.getOpcode() != ISD::FMUL)
16775 return SDValue();
16776
16777 SDValue ConstVec = Op->getOperand(1);
16778 if (!isa<BuildVectorSDNode>(ConstVec))
16779 return SDValue();
16780
16781 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
16782 uint32_t FloatBits = FloatTy.getSizeInBits();
16783 MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
16784 uint32_t IntBits = IntTy.getSizeInBits();
16785 unsigned NumLanes = Op.getValueType().getVectorNumElements();
16786 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16787 // These instructions only exist converting from f32 to i32. We can handle
16788 // smaller integers by generating an extra truncate, but larger ones would
16789 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
16790 // these intructions only support v2i32/v4i32 types.
16791 return SDValue();
16792 }
16793
16794 BitVector UndefElements;
16795 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
16796 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
16797 if (C == -1 || C == 0 || C > 32)
16798 return SDValue();
16799
16800 SDLoc dl(N);
16801 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
16802 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16803 Intrinsic::arm_neon_vcvtfp2fxu;
16804 SDValue FixConv = DAG.getNode(
16805 ISD::INTRINSIC_WO_CHAIN, dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
16806 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0),
16807 DAG.getConstant(C, dl, MVT::i32));
16808
16809 if (IntBits < FloatBits)
16810 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv);
16811
16812 return FixConv;
16813 }
16814
PerformFAddVSelectCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)16815 static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG,
16816 const ARMSubtarget *Subtarget) {
16817 if (!Subtarget->hasMVEFloatOps())
16818 return SDValue();
16819
16820 // Turn (fadd x, (vselect c, y, -0.0)) into (vselect c, (fadd x, y), x)
16821 // The second form can be more easily turned into a predicated vadd, and
16822 // possibly combined into a fma to become a predicated vfma.
16823 SDValue Op0 = N->getOperand(0);
16824 SDValue Op1 = N->getOperand(1);
16825 EVT VT = N->getValueType(0);
16826 SDLoc DL(N);
16827
16828 // The identity element for a fadd is -0.0 or +0.0 when the nsz flag is set,
16829 // which these VMOV's represent.
16830 auto isIdentitySplat = [&](SDValue Op, bool NSZ) {
16831 if (Op.getOpcode() != ISD::BITCAST ||
16832 Op.getOperand(0).getOpcode() != ARMISD::VMOVIMM)
16833 return false;
16834 uint64_t ImmVal = Op.getOperand(0).getConstantOperandVal(0);
16835 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16836 return true;
16837 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16838 return true;
16839 return false;
16840 };
16841
16842 if (Op0.getOpcode() == ISD::VSELECT && Op1.getOpcode() != ISD::VSELECT)
16843 std::swap(Op0, Op1);
16844
16845 if (Op1.getOpcode() != ISD::VSELECT)
16846 return SDValue();
16847
16848 SDNodeFlags FaddFlags = N->getFlags();
16849 bool NSZ = FaddFlags.hasNoSignedZeros();
16850 if (!isIdentitySplat(Op1.getOperand(2), NSZ))
16851 return SDValue();
16852
16853 SDValue FAdd =
16854 DAG.getNode(ISD::FADD, DL, VT, Op0, Op1.getOperand(1), FaddFlags);
16855 return DAG.getNode(ISD::VSELECT, DL, VT, Op1.getOperand(0), FAdd, Op0, FaddFlags);
16856 }
16857
16858 /// PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
16859 /// can replace combinations of VCVT (integer to floating-point) and VDIV
16860 /// when the VDIV has a constant operand that is a power of 2.
16861 ///
16862 /// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
16863 /// vcvt.f32.s32 d16, d16
16864 /// vdiv.f32 d16, d17, d16
16865 /// becomes:
16866 /// vcvt.f32.s32 d16, d16, #3
PerformVDIVCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)16867 static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG,
16868 const ARMSubtarget *Subtarget) {
16869 if (!Subtarget->hasNEON())
16870 return SDValue();
16871
16872 SDValue Op = N->getOperand(0);
16873 unsigned OpOpcode = Op.getNode()->getOpcode();
16874 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple() ||
16875 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
16876 return SDValue();
16877
16878 SDValue ConstVec = N->getOperand(1);
16879 if (!isa<BuildVectorSDNode>(ConstVec))
16880 return SDValue();
16881
16882 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
16883 uint32_t FloatBits = FloatTy.getSizeInBits();
16884 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
16885 uint32_t IntBits = IntTy.getSizeInBits();
16886 unsigned NumLanes = Op.getValueType().getVectorNumElements();
16887 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16888 // These instructions only exist converting from i32 to f32. We can handle
16889 // smaller integers by generating an extra extend, but larger ones would
16890 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
16891 // these intructions only support v2i32/v4i32 types.
16892 return SDValue();
16893 }
16894
16895 BitVector UndefElements;
16896 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
16897 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
16898 if (C == -1 || C == 0 || C > 32)
16899 return SDValue();
16900
16901 SDLoc dl(N);
16902 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
16903 SDValue ConvInput = Op.getOperand(0);
16904 if (IntBits < FloatBits)
16905 ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
16906 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
16907 ConvInput);
16908
16909 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
16910 Intrinsic::arm_neon_vcvtfxu2fp;
16911 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl,
16912 Op.getValueType(),
16913 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32),
16914 ConvInput, DAG.getConstant(C, dl, MVT::i32));
16915 }
16916
PerformVECREDUCE_ADDCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)16917 static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
16918 const ARMSubtarget *ST) {
16919 if (!ST->hasMVEIntegerOps())
16920 return SDValue();
16921
16922 assert(N->getOpcode() == ISD::VECREDUCE_ADD);
16923 EVT ResVT = N->getValueType(0);
16924 SDValue N0 = N->getOperand(0);
16925 SDLoc dl(N);
16926
16927 // Try to turn vecreduce_add(add(x, y)) into vecreduce(x) + vecreduce(y)
16928 if (ResVT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
16929 (N0.getValueType() == MVT::v4i32 || N0.getValueType() == MVT::v8i16 ||
16930 N0.getValueType() == MVT::v16i8)) {
16931 SDValue Red0 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(0));
16932 SDValue Red1 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(1));
16933 return DAG.getNode(ISD::ADD, dl, ResVT, Red0, Red1);
16934 }
16935
16936 // We are looking for something that will have illegal types if left alone,
16937 // but that we can convert to a single instruction under MVE. For example
16938 // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
16939 // or
16940 // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
16941
16942 // The legal cases are:
16943 // VADDV u/s 8/16/32
16944 // VMLAV u/s 8/16/32
16945 // VADDLV u/s 32
16946 // VMLALV u/s 16/32
16947
16948 // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
16949 // extend it and use v4i32 instead.
16950 auto ExtTypeMatches = [](SDValue A, ArrayRef<MVT> ExtTypes) {
16951 EVT AVT = A.getValueType();
16952 return any_of(ExtTypes, [&](MVT Ty) {
16953 return AVT.getVectorNumElements() == Ty.getVectorNumElements() &&
16954 AVT.bitsLE(Ty);
16955 });
16956 };
16957 auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
16958 EVT AVT = A.getValueType();
16959 if (!AVT.is128BitVector())
16960 A = DAG.getNode(ExtendCode, dl,
16961 AVT.changeVectorElementType(MVT::getIntegerVT(
16962 128 / AVT.getVectorMinNumElements())),
16963 A);
16964 return A;
16965 };
16966 auto IsVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes) {
16967 if (ResVT != RetTy || N0->getOpcode() != ExtendCode)
16968 return SDValue();
16969 SDValue A = N0->getOperand(0);
16970 if (ExtTypeMatches(A, ExtTypes))
16971 return ExtendIfNeeded(A, ExtendCode);
16972 return SDValue();
16973 };
16974 auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode,
16975 ArrayRef<MVT> ExtTypes, SDValue &Mask) {
16976 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
16977 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode()))
16978 return SDValue();
16979 Mask = N0->getOperand(0);
16980 SDValue Ext = N0->getOperand(1);
16981 if (Ext->getOpcode() != ExtendCode)
16982 return SDValue();
16983 SDValue A = Ext->getOperand(0);
16984 if (ExtTypeMatches(A, ExtTypes))
16985 return ExtendIfNeeded(A, ExtendCode);
16986 return SDValue();
16987 };
16988 auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
16989 SDValue &A, SDValue &B) {
16990 // For a vmla we are trying to match a larger pattern:
16991 // ExtA = sext/zext A
16992 // ExtB = sext/zext B
16993 // Mul = mul ExtA, ExtB
16994 // vecreduce.add Mul
16995 // There might also be en extra extend between the mul and the addreduce, so
16996 // long as the bitwidth is high enough to make them equivalent (for example
16997 // original v8i16 might be mul at v8i32 and the reduce happens at v8i64).
16998 if (ResVT != RetTy)
16999 return false;
17000 SDValue Mul = N0;
17001 if (Mul->getOpcode() == ExtendCode &&
17002 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17003 ResVT.getScalarSizeInBits())
17004 Mul = Mul->getOperand(0);
17005 if (Mul->getOpcode() != ISD::MUL)
17006 return false;
17007 SDValue ExtA = Mul->getOperand(0);
17008 SDValue ExtB = Mul->getOperand(1);
17009 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17010 return false;
17011 A = ExtA->getOperand(0);
17012 B = ExtB->getOperand(0);
17013 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17014 A = ExtendIfNeeded(A, ExtendCode);
17015 B = ExtendIfNeeded(B, ExtendCode);
17016 return true;
17017 }
17018 return false;
17019 };
17020 auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17021 SDValue &A, SDValue &B, SDValue &Mask) {
17022 // Same as the pattern above with a select for the zero predicated lanes
17023 // ExtA = sext/zext A
17024 // ExtB = sext/zext B
17025 // Mul = mul ExtA, ExtB
17026 // N0 = select Mask, Mul, 0
17027 // vecreduce.add N0
17028 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17029 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode()))
17030 return false;
17031 Mask = N0->getOperand(0);
17032 SDValue Mul = N0->getOperand(1);
17033 if (Mul->getOpcode() == ExtendCode &&
17034 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17035 ResVT.getScalarSizeInBits())
17036 Mul = Mul->getOperand(0);
17037 if (Mul->getOpcode() != ISD::MUL)
17038 return false;
17039 SDValue ExtA = Mul->getOperand(0);
17040 SDValue ExtB = Mul->getOperand(1);
17041 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17042 return false;
17043 A = ExtA->getOperand(0);
17044 B = ExtB->getOperand(0);
17045 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17046 A = ExtendIfNeeded(A, ExtendCode);
17047 B = ExtendIfNeeded(B, ExtendCode);
17048 return true;
17049 }
17050 return false;
17051 };
17052 auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
17053 // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64
17054 // reductions. The operands are extended with MVEEXT, but as they are
17055 // reductions the lane orders do not matter. MVEEXT may be combined with
17056 // loads to produce two extending loads, or else they will be expanded to
17057 // VREV/VMOVL.
17058 EVT VT = Ops[0].getValueType();
17059 if (VT == MVT::v16i8) {
17060 assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
17061 "Unexpected illegal long reduction opcode");
17062 bool IsUnsigned = Opcode == ARMISD::VMLALVu;
17063
17064 SDValue Ext0 =
17065 DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
17066 DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17067 SDValue Ext1 =
17068 DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
17069 DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17070
17071 SDValue MLA0 = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
17072 Ext0, Ext1);
17073 SDValue MLA1 =
17074 DAG.getNode(IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, dl,
17075 DAG.getVTList(MVT::i32, MVT::i32), MLA0, MLA0.getValue(1),
17076 Ext0.getValue(1), Ext1.getValue(1));
17077 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, MLA1, MLA1.getValue(1));
17078 }
17079 SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
17080 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node,
17081 SDValue(Node.getNode(), 1));
17082 };
17083
17084 SDValue A, B;
17085 SDValue Mask;
17086 if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17087 return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B);
17088 if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17089 return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
17090 if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17091 A, B))
17092 return Create64bitNode(ARMISD::VMLALVs, {A, B});
17093 if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17094 A, B))
17095 return Create64bitNode(ARMISD::VMLALVu, {A, B});
17096 if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
17097 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17098 DAG.getNode(ARMISD::VMLAVs, dl, MVT::i32, A, B));
17099 if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B))
17100 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17101 DAG.getNode(ARMISD::VMLAVu, dl, MVT::i32, A, B));
17102
17103 if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17104 Mask))
17105 return DAG.getNode(ARMISD::VMLAVps, dl, ResVT, A, B, Mask);
17106 if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17107 Mask))
17108 return DAG.getNode(ARMISD::VMLAVpu, dl, ResVT, A, B, Mask);
17109 if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17110 Mask))
17111 return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
17112 if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17113 Mask))
17114 return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
17115 if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask))
17116 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17117 DAG.getNode(ARMISD::VMLAVps, dl, MVT::i32, A, B, Mask));
17118 if (IsPredVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B, Mask))
17119 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17120 DAG.getNode(ARMISD::VMLAVpu, dl, MVT::i32, A, B, Mask));
17121
17122 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
17123 return DAG.getNode(ARMISD::VADDVs, dl, ResVT, A);
17124 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
17125 return DAG.getNode(ARMISD::VADDVu, dl, ResVT, A);
17126 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}))
17127 return Create64bitNode(ARMISD::VADDLVs, {A});
17128 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
17129 return Create64bitNode(ARMISD::VADDLVu, {A});
17130 if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}))
17131 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17132 DAG.getNode(ARMISD::VADDVs, dl, MVT::i32, A));
17133 if (SDValue A = IsVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}))
17134 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17135 DAG.getNode(ARMISD::VADDVu, dl, MVT::i32, A));
17136
17137 if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17138 return DAG.getNode(ARMISD::VADDVps, dl, ResVT, A, Mask);
17139 if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17140 return DAG.getNode(ARMISD::VADDVpu, dl, ResVT, A, Mask);
17141 if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask))
17142 return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
17143 if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask))
17144 return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
17145 if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask))
17146 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17147 DAG.getNode(ARMISD::VADDVps, dl, MVT::i32, A, Mask));
17148 if (SDValue A = IsPredVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, Mask))
17149 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17150 DAG.getNode(ARMISD::VADDVpu, dl, MVT::i32, A, Mask));
17151
17152 // Some complications. We can get a case where the two inputs of the mul are
17153 // the same, then the output sext will have been helpfully converted to a
17154 // zext. Turn it back.
17155 SDValue Op = N0;
17156 if (Op->getOpcode() == ISD::VSELECT)
17157 Op = Op->getOperand(1);
17158 if (Op->getOpcode() == ISD::ZERO_EXTEND &&
17159 Op->getOperand(0)->getOpcode() == ISD::MUL) {
17160 SDValue Mul = Op->getOperand(0);
17161 if (Mul->getOperand(0) == Mul->getOperand(1) &&
17162 Mul->getOperand(0)->getOpcode() == ISD::SIGN_EXTEND) {
17163 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, N0->getValueType(0), Mul);
17164 if (Op != N0)
17165 Ext = DAG.getNode(ISD::VSELECT, dl, N0->getValueType(0),
17166 N0->getOperand(0), Ext, N0->getOperand(2));
17167 return DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, Ext);
17168 }
17169 }
17170
17171 return SDValue();
17172 }
17173
PerformVMOVNCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)17174 static SDValue PerformVMOVNCombine(SDNode *N,
17175 TargetLowering::DAGCombinerInfo &DCI) {
17176 SDValue Op0 = N->getOperand(0);
17177 SDValue Op1 = N->getOperand(1);
17178 unsigned IsTop = N->getConstantOperandVal(2);
17179
17180 // VMOVNT a undef -> a
17181 // VMOVNB a undef -> a
17182 // VMOVNB undef a -> a
17183 if (Op1->isUndef())
17184 return Op0;
17185 if (Op0->isUndef() && !IsTop)
17186 return Op1;
17187
17188 // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b)
17189 // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b)
17190 if ((Op1->getOpcode() == ARMISD::VQMOVNs ||
17191 Op1->getOpcode() == ARMISD::VQMOVNu) &&
17192 Op1->getConstantOperandVal(2) == 0)
17193 return DCI.DAG.getNode(Op1->getOpcode(), SDLoc(Op1), N->getValueType(0),
17194 Op0, Op1->getOperand(1), N->getOperand(2));
17195
17196 // Only the bottom lanes from Qm (Op1) and either the top or bottom lanes from
17197 // Qd (Op0) are demanded from a VMOVN, depending on whether we are inserting
17198 // into the top or bottom lanes.
17199 unsigned NumElts = N->getValueType(0).getVectorNumElements();
17200 APInt Op1DemandedElts = APInt::getSplat(NumElts, APInt::getLowBitsSet(2, 1));
17201 APInt Op0DemandedElts =
17202 IsTop ? Op1DemandedElts
17203 : APInt::getSplat(NumElts, APInt::getHighBitsSet(2, 1));
17204
17205 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17206 if (TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, DCI))
17207 return SDValue(N, 0);
17208 if (TLI.SimplifyDemandedVectorElts(Op1, Op1DemandedElts, DCI))
17209 return SDValue(N, 0);
17210
17211 return SDValue();
17212 }
17213
PerformVQMOVNCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)17214 static SDValue PerformVQMOVNCombine(SDNode *N,
17215 TargetLowering::DAGCombinerInfo &DCI) {
17216 SDValue Op0 = N->getOperand(0);
17217 unsigned IsTop = N->getConstantOperandVal(2);
17218
17219 unsigned NumElts = N->getValueType(0).getVectorNumElements();
17220 APInt Op0DemandedElts =
17221 APInt::getSplat(NumElts, IsTop ? APInt::getLowBitsSet(2, 1)
17222 : APInt::getHighBitsSet(2, 1));
17223
17224 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17225 if (TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, DCI))
17226 return SDValue(N, 0);
17227 return SDValue();
17228 }
17229
PerformLongShiftCombine(SDNode * N,SelectionDAG & DAG)17230 static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
17231 SDLoc DL(N);
17232 SDValue Op0 = N->getOperand(0);
17233 SDValue Op1 = N->getOperand(1);
17234
17235 // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from
17236 // uses of the intrinsics.
17237 if (auto C = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
17238 int ShiftAmt = C->getSExtValue();
17239 if (ShiftAmt == 0) {
17240 SDValue Merge = DAG.getMergeValues({Op0, Op1}, DL);
17241 DAG.ReplaceAllUsesWith(N, Merge.getNode());
17242 return SDValue();
17243 }
17244
17245 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17246 unsigned NewOpcode =
17247 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL;
17248 SDValue NewShift = DAG.getNode(NewOpcode, DL, N->getVTList(), Op0, Op1,
17249 DAG.getConstant(-ShiftAmt, DL, MVT::i32));
17250 DAG.ReplaceAllUsesWith(N, NewShift.getNode());
17251 return NewShift;
17252 }
17253 }
17254
17255 return SDValue();
17256 }
17257
17258 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
PerformIntrinsicCombine(SDNode * N,DAGCombinerInfo & DCI) const17259 SDValue ARMTargetLowering::PerformIntrinsicCombine(SDNode *N,
17260 DAGCombinerInfo &DCI) const {
17261 SelectionDAG &DAG = DCI.DAG;
17262 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
17263 switch (IntNo) {
17264 default:
17265 // Don't do anything for most intrinsics.
17266 break;
17267
17268 // Vector shifts: check for immediate versions and lower them.
17269 // Note: This is done during DAG combining instead of DAG legalizing because
17270 // the build_vectors for 64-bit vector element shift counts are generally
17271 // not legal, and it is hard to see their values after they get legalized to
17272 // loads from a constant pool.
17273 case Intrinsic::arm_neon_vshifts:
17274 case Intrinsic::arm_neon_vshiftu:
17275 case Intrinsic::arm_neon_vrshifts:
17276 case Intrinsic::arm_neon_vrshiftu:
17277 case Intrinsic::arm_neon_vrshiftn:
17278 case Intrinsic::arm_neon_vqshifts:
17279 case Intrinsic::arm_neon_vqshiftu:
17280 case Intrinsic::arm_neon_vqshiftsu:
17281 case Intrinsic::arm_neon_vqshiftns:
17282 case Intrinsic::arm_neon_vqshiftnu:
17283 case Intrinsic::arm_neon_vqshiftnsu:
17284 case Intrinsic::arm_neon_vqrshiftns:
17285 case Intrinsic::arm_neon_vqrshiftnu:
17286 case Intrinsic::arm_neon_vqrshiftnsu: {
17287 EVT VT = N->getOperand(1).getValueType();
17288 int64_t Cnt;
17289 unsigned VShiftOpc = 0;
17290
17291 switch (IntNo) {
17292 case Intrinsic::arm_neon_vshifts:
17293 case Intrinsic::arm_neon_vshiftu:
17294 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) {
17295 VShiftOpc = ARMISD::VSHLIMM;
17296 break;
17297 }
17298 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) {
17299 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? ARMISD::VSHRsIMM
17300 : ARMISD::VSHRuIMM);
17301 break;
17302 }
17303 return SDValue();
17304
17305 case Intrinsic::arm_neon_vrshifts:
17306 case Intrinsic::arm_neon_vrshiftu:
17307 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt))
17308 break;
17309 return SDValue();
17310
17311 case Intrinsic::arm_neon_vqshifts:
17312 case Intrinsic::arm_neon_vqshiftu:
17313 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
17314 break;
17315 return SDValue();
17316
17317 case Intrinsic::arm_neon_vqshiftsu:
17318 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
17319 break;
17320 llvm_unreachable("invalid shift count for vqshlu intrinsic");
17321
17322 case Intrinsic::arm_neon_vrshiftn:
17323 case Intrinsic::arm_neon_vqshiftns:
17324 case Intrinsic::arm_neon_vqshiftnu:
17325 case Intrinsic::arm_neon_vqshiftnsu:
17326 case Intrinsic::arm_neon_vqrshiftns:
17327 case Intrinsic::arm_neon_vqrshiftnu:
17328 case Intrinsic::arm_neon_vqrshiftnsu:
17329 // Narrowing shifts require an immediate right shift.
17330 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt))
17331 break;
17332 llvm_unreachable("invalid shift count for narrowing vector shift "
17333 "intrinsic");
17334
17335 default:
17336 llvm_unreachable("unhandled vector shift");
17337 }
17338
17339 switch (IntNo) {
17340 case Intrinsic::arm_neon_vshifts:
17341 case Intrinsic::arm_neon_vshiftu:
17342 // Opcode already set above.
17343 break;
17344 case Intrinsic::arm_neon_vrshifts:
17345 VShiftOpc = ARMISD::VRSHRsIMM;
17346 break;
17347 case Intrinsic::arm_neon_vrshiftu:
17348 VShiftOpc = ARMISD::VRSHRuIMM;
17349 break;
17350 case Intrinsic::arm_neon_vrshiftn:
17351 VShiftOpc = ARMISD::VRSHRNIMM;
17352 break;
17353 case Intrinsic::arm_neon_vqshifts:
17354 VShiftOpc = ARMISD::VQSHLsIMM;
17355 break;
17356 case Intrinsic::arm_neon_vqshiftu:
17357 VShiftOpc = ARMISD::VQSHLuIMM;
17358 break;
17359 case Intrinsic::arm_neon_vqshiftsu:
17360 VShiftOpc = ARMISD::VQSHLsuIMM;
17361 break;
17362 case Intrinsic::arm_neon_vqshiftns:
17363 VShiftOpc = ARMISD::VQSHRNsIMM;
17364 break;
17365 case Intrinsic::arm_neon_vqshiftnu:
17366 VShiftOpc = ARMISD::VQSHRNuIMM;
17367 break;
17368 case Intrinsic::arm_neon_vqshiftnsu:
17369 VShiftOpc = ARMISD::VQSHRNsuIMM;
17370 break;
17371 case Intrinsic::arm_neon_vqrshiftns:
17372 VShiftOpc = ARMISD::VQRSHRNsIMM;
17373 break;
17374 case Intrinsic::arm_neon_vqrshiftnu:
17375 VShiftOpc = ARMISD::VQRSHRNuIMM;
17376 break;
17377 case Intrinsic::arm_neon_vqrshiftnsu:
17378 VShiftOpc = ARMISD::VQRSHRNsuIMM;
17379 break;
17380 }
17381
17382 SDLoc dl(N);
17383 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
17384 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32));
17385 }
17386
17387 case Intrinsic::arm_neon_vshiftins: {
17388 EVT VT = N->getOperand(1).getValueType();
17389 int64_t Cnt;
17390 unsigned VShiftOpc = 0;
17391
17392 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt))
17393 VShiftOpc = ARMISD::VSLIIMM;
17394 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt))
17395 VShiftOpc = ARMISD::VSRIIMM;
17396 else {
17397 llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
17398 }
17399
17400 SDLoc dl(N);
17401 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
17402 N->getOperand(1), N->getOperand(2),
17403 DAG.getConstant(Cnt, dl, MVT::i32));
17404 }
17405
17406 case Intrinsic::arm_neon_vqrshifts:
17407 case Intrinsic::arm_neon_vqrshiftu:
17408 // No immediate versions of these to check for.
17409 break;
17410
17411 case Intrinsic::arm_mve_vqdmlah:
17412 case Intrinsic::arm_mve_vqdmlash:
17413 case Intrinsic::arm_mve_vqrdmlah:
17414 case Intrinsic::arm_mve_vqrdmlash:
17415 case Intrinsic::arm_mve_vmla_n_predicated:
17416 case Intrinsic::arm_mve_vmlas_n_predicated:
17417 case Intrinsic::arm_mve_vqdmlah_predicated:
17418 case Intrinsic::arm_mve_vqdmlash_predicated:
17419 case Intrinsic::arm_mve_vqrdmlah_predicated:
17420 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17421 // These intrinsics all take an i32 scalar operand which is narrowed to the
17422 // size of a single lane of the vector type they return. So we don't need
17423 // any bits of that operand above that point, which allows us to eliminate
17424 // uxth/sxth.
17425 unsigned BitWidth = N->getValueType(0).getScalarSizeInBits();
17426 APInt DemandedMask = APInt::getLowBitsSet(32, BitWidth);
17427 if (SimplifyDemandedBits(N->getOperand(3), DemandedMask, DCI))
17428 return SDValue();
17429 break;
17430 }
17431
17432 case Intrinsic::arm_mve_minv:
17433 case Intrinsic::arm_mve_maxv:
17434 case Intrinsic::arm_mve_minav:
17435 case Intrinsic::arm_mve_maxav:
17436 case Intrinsic::arm_mve_minv_predicated:
17437 case Intrinsic::arm_mve_maxv_predicated:
17438 case Intrinsic::arm_mve_minav_predicated:
17439 case Intrinsic::arm_mve_maxav_predicated: {
17440 // These intrinsics all take an i32 scalar operand which is narrowed to the
17441 // size of a single lane of the vector type they take as the other input.
17442 unsigned BitWidth = N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17443 APInt DemandedMask = APInt::getLowBitsSet(32, BitWidth);
17444 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
17445 return SDValue();
17446 break;
17447 }
17448
17449 case Intrinsic::arm_mve_addv: {
17450 // Turn this intrinsic straight into the appropriate ARMISD::VADDV node,
17451 // which allow PerformADDVecReduce to turn it into VADDLV when possible.
17452 bool Unsigned = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
17453 unsigned Opc = Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
17454 return DAG.getNode(Opc, SDLoc(N), N->getVTList(), N->getOperand(1));
17455 }
17456
17457 case Intrinsic::arm_mve_addlv:
17458 case Intrinsic::arm_mve_addlv_predicated: {
17459 // Same for these, but ARMISD::VADDLV has to be followed by a BUILD_PAIR
17460 // which recombines the two outputs into an i64
17461 bool Unsigned = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
17462 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17463 (Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
17464 (Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
17465
17466 SmallVector<SDValue, 4> Ops;
17467 for (unsigned i = 1, e = N->getNumOperands(); i < e; i++)
17468 if (i != 2) // skip the unsigned flag
17469 Ops.push_back(N->getOperand(i));
17470
17471 SDLoc dl(N);
17472 SDValue val = DAG.getNode(Opc, dl, {MVT::i32, MVT::i32}, Ops);
17473 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, val.getValue(0),
17474 val.getValue(1));
17475 }
17476 }
17477
17478 return SDValue();
17479 }
17480
17481 /// PerformShiftCombine - Checks for immediate versions of vector shifts and
17482 /// lowers them. As with the vector shift intrinsics, this is done during DAG
17483 /// combining instead of DAG legalizing because the build_vectors for 64-bit
17484 /// vector element shift counts are generally not legal, and it is hard to see
17485 /// their values after they get legalized to loads from a constant pool.
PerformShiftCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * ST)17486 static SDValue PerformShiftCombine(SDNode *N,
17487 TargetLowering::DAGCombinerInfo &DCI,
17488 const ARMSubtarget *ST) {
17489 SelectionDAG &DAG = DCI.DAG;
17490 EVT VT = N->getValueType(0);
17491
17492 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 &&
17493 N->getOperand(0)->getOpcode() == ISD::AND &&
17494 N->getOperand(0)->hasOneUse()) {
17495 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
17496 return SDValue();
17497 // Look for the pattern (shl (and x, AndMask), ShiftAmt). This doesn't
17498 // usually show up because instcombine prefers to canonicalize it to
17499 // (and (shl x, ShiftAmt) (shl AndMask, ShiftAmt)), but the shift can come
17500 // out of GEP lowering in some cases.
17501 SDValue N0 = N->getOperand(0);
17502 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
17503 if (!ShiftAmtNode)
17504 return SDValue();
17505 uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue());
17506 ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(N0->getOperand(1));
17507 if (!AndMaskNode)
17508 return SDValue();
17509 uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue());
17510 // Don't transform uxtb/uxth.
17511 if (AndMask == 255 || AndMask == 65535)
17512 return SDValue();
17513 if (isMask_32(AndMask)) {
17514 uint32_t MaskedBits = countLeadingZeros(AndMask);
17515 if (MaskedBits > ShiftAmt) {
17516 SDLoc DL(N);
17517 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
17518 DAG.getConstant(MaskedBits, DL, MVT::i32));
17519 return DAG.getNode(
17520 ISD::SRL, DL, MVT::i32, SHL,
17521 DAG.getConstant(MaskedBits - ShiftAmt, DL, MVT::i32));
17522 }
17523 }
17524 }
17525
17526 // Nothing to be done for scalar shifts.
17527 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17528 if (!VT.isVector() || !TLI.isTypeLegal(VT))
17529 return SDValue();
17530 if (ST->hasMVEIntegerOps())
17531 return SDValue();
17532
17533 int64_t Cnt;
17534
17535 switch (N->getOpcode()) {
17536 default: llvm_unreachable("unexpected shift opcode");
17537
17538 case ISD::SHL:
17539 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) {
17540 SDLoc dl(N);
17541 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
17542 DAG.getConstant(Cnt, dl, MVT::i32));
17543 }
17544 break;
17545
17546 case ISD::SRA:
17547 case ISD::SRL:
17548 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
17549 unsigned VShiftOpc =
17550 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
17551 SDLoc dl(N);
17552 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
17553 DAG.getConstant(Cnt, dl, MVT::i32));
17554 }
17555 }
17556 return SDValue();
17557 }
17558
17559 // Look for a sign/zero/fpextend extend of a larger than legal load. This can be
17560 // split into multiple extending loads, which are simpler to deal with than an
17561 // arbitrary extend. For fp extends we use an integer extending load and a VCVTL
17562 // to convert the type to an f32.
PerformSplittingToWideningLoad(SDNode * N,SelectionDAG & DAG)17563 static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
17564 SDValue N0 = N->getOperand(0);
17565 if (N0.getOpcode() != ISD::LOAD)
17566 return SDValue();
17567 LoadSDNode *LD = cast<LoadSDNode>(N0.getNode());
17568 if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() ||
17569 LD->getExtensionType() != ISD::NON_EXTLOAD)
17570 return SDValue();
17571 EVT FromVT = LD->getValueType(0);
17572 EVT ToVT = N->getValueType(0);
17573 if (!ToVT.isVector())
17574 return SDValue();
17575 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
17576 EVT ToEltVT = ToVT.getVectorElementType();
17577 EVT FromEltVT = FromVT.getVectorElementType();
17578
17579 unsigned NumElements = 0;
17580 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17581 NumElements = 4;
17582 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17583 NumElements = 4;
17584 if (NumElements == 0 ||
17585 (FromEltVT != MVT::f16 && FromVT.getVectorNumElements() == NumElements) ||
17586 FromVT.getVectorNumElements() % NumElements != 0 ||
17587 !isPowerOf2_32(NumElements))
17588 return SDValue();
17589
17590 LLVMContext &C = *DAG.getContext();
17591 SDLoc DL(LD);
17592 // Details about the old load
17593 SDValue Ch = LD->getChain();
17594 SDValue BasePtr = LD->getBasePtr();
17595 Align Alignment = LD->getOriginalAlign();
17596 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
17597 AAMDNodes AAInfo = LD->getAAInfo();
17598
17599 ISD::LoadExtType NewExtType =
17600 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
17601 SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
17602 EVT NewFromVT = EVT::getVectorVT(
17603 C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
17604 EVT NewToVT = EVT::getVectorVT(
17605 C, EVT::getIntegerVT(C, ToEltVT.getScalarSizeInBits()), NumElements);
17606
17607 SmallVector<SDValue, 4> Loads;
17608 SmallVector<SDValue, 4> Chains;
17609 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
17610 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
17611 SDValue NewPtr =
17612 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset));
17613
17614 SDValue NewLoad =
17615 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
17616 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17617 Alignment, MMOFlags, AAInfo);
17618 Loads.push_back(NewLoad);
17619 Chains.push_back(SDValue(NewLoad.getNode(), 1));
17620 }
17621
17622 // Float truncs need to extended with VCVTB's into their floating point types.
17623 if (FromEltVT == MVT::f16) {
17624 SmallVector<SDValue, 4> Extends;
17625
17626 for (unsigned i = 0; i < Loads.size(); i++) {
17627 SDValue LoadBC =
17628 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v8f16, Loads[i]);
17629 SDValue FPExt = DAG.getNode(ARMISD::VCVTL, DL, MVT::v4f32, LoadBC,
17630 DAG.getConstant(0, DL, MVT::i32));
17631 Extends.push_back(FPExt);
17632 }
17633
17634 Loads = Extends;
17635 }
17636
17637 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
17638 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
17639 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Loads);
17640 }
17641
17642 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
17643 /// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
PerformExtendCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)17644 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
17645 const ARMSubtarget *ST) {
17646 SDValue N0 = N->getOperand(0);
17647
17648 // Check for sign- and zero-extensions of vector extract operations of 8- and
17649 // 16-bit vector elements. NEON and MVE support these directly. They are
17650 // handled during DAG combining because type legalization will promote them
17651 // to 32-bit types and it is messy to recognize the operations after that.
17652 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17653 N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17654 SDValue Vec = N0.getOperand(0);
17655 SDValue Lane = N0.getOperand(1);
17656 EVT VT = N->getValueType(0);
17657 EVT EltVT = N0.getValueType();
17658 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17659
17660 if (VT == MVT::i32 &&
17661 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17662 TLI.isTypeLegal(Vec.getValueType()) &&
17663 isa<ConstantSDNode>(Lane)) {
17664
17665 unsigned Opc = 0;
17666 switch (N->getOpcode()) {
17667 default: llvm_unreachable("unexpected opcode");
17668 case ISD::SIGN_EXTEND:
17669 Opc = ARMISD::VGETLANEs;
17670 break;
17671 case ISD::ZERO_EXTEND:
17672 case ISD::ANY_EXTEND:
17673 Opc = ARMISD::VGETLANEu;
17674 break;
17675 }
17676 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane);
17677 }
17678 }
17679
17680 if (ST->hasMVEIntegerOps())
17681 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
17682 return NewLoad;
17683
17684 return SDValue();
17685 }
17686
PerformFPExtendCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)17687 static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
17688 const ARMSubtarget *ST) {
17689 if (ST->hasMVEFloatOps())
17690 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
17691 return NewLoad;
17692
17693 return SDValue();
17694 }
17695
17696 // Lower smin(smax(x, C1), C2) to ssat or usat, if they have saturating
17697 // constant bounds.
PerformMinMaxToSatCombine(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)17698 static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG,
17699 const ARMSubtarget *Subtarget) {
17700 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17701 !Subtarget->isThumb2())
17702 return SDValue();
17703
17704 EVT VT = Op.getValueType();
17705 SDValue Op0 = Op.getOperand(0);
17706
17707 if (VT != MVT::i32 ||
17708 (Op0.getOpcode() != ISD::SMIN && Op0.getOpcode() != ISD::SMAX) ||
17709 !isa<ConstantSDNode>(Op.getOperand(1)) ||
17710 !isa<ConstantSDNode>(Op0.getOperand(1)))
17711 return SDValue();
17712
17713 SDValue Min = Op;
17714 SDValue Max = Op0;
17715 SDValue Input = Op0.getOperand(0);
17716 if (Min.getOpcode() == ISD::SMAX)
17717 std::swap(Min, Max);
17718
17719 APInt MinC = Min.getConstantOperandAPInt(1);
17720 APInt MaxC = Max.getConstantOperandAPInt(1);
17721
17722 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX ||
17723 !(MinC + 1).isPowerOf2())
17724 return SDValue();
17725
17726 SDLoc DL(Op);
17727 if (MinC == ~MaxC)
17728 return DAG.getNode(ARMISD::SSAT, DL, VT, Input,
17729 DAG.getConstant(MinC.countTrailingOnes(), DL, VT));
17730 if (MaxC == 0)
17731 return DAG.getNode(ARMISD::USAT, DL, VT, Input,
17732 DAG.getConstant(MinC.countTrailingOnes(), DL, VT));
17733
17734 return SDValue();
17735 }
17736
17737 /// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
17738 /// saturates.
PerformMinMaxCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)17739 static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
17740 const ARMSubtarget *ST) {
17741 EVT VT = N->getValueType(0);
17742 SDValue N0 = N->getOperand(0);
17743
17744 if (VT == MVT::i32)
17745 return PerformMinMaxToSatCombine(SDValue(N, 0), DAG, ST);
17746
17747 if (!ST->hasMVEIntegerOps())
17748 return SDValue();
17749
17750 if (SDValue V = PerformVQDMULHCombine(N, DAG))
17751 return V;
17752
17753 if (VT != MVT::v4i32 && VT != MVT::v8i16)
17754 return SDValue();
17755
17756 auto IsSignedSaturate = [&](SDNode *Min, SDNode *Max) {
17757 // Check one is a smin and the other is a smax
17758 if (Min->getOpcode() != ISD::SMIN)
17759 std::swap(Min, Max);
17760 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX)
17761 return false;
17762
17763 APInt SaturateC;
17764 if (VT == MVT::v4i32)
17765 SaturateC = APInt(32, (1 << 15) - 1, true);
17766 else //if (VT == MVT::v8i16)
17767 SaturateC = APInt(16, (1 << 7) - 1, true);
17768
17769 APInt MinC, MaxC;
17770 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) ||
17771 MinC != SaturateC)
17772 return false;
17773 if (!ISD::isConstantSplatVector(Max->getOperand(1).getNode(), MaxC) ||
17774 MaxC != ~SaturateC)
17775 return false;
17776 return true;
17777 };
17778
17779 if (IsSignedSaturate(N, N0.getNode())) {
17780 SDLoc DL(N);
17781 MVT ExtVT, HalfVT;
17782 if (VT == MVT::v4i32) {
17783 HalfVT = MVT::v8i16;
17784 ExtVT = MVT::v4i16;
17785 } else { // if (VT == MVT::v8i16)
17786 HalfVT = MVT::v16i8;
17787 ExtVT = MVT::v8i8;
17788 }
17789
17790 // Create a VQMOVNB with undef top lanes, then signed extended into the top
17791 // half. That extend will hopefully be removed if only the bottom bits are
17792 // demanded (though a truncating store, for example).
17793 SDValue VQMOVN =
17794 DAG.getNode(ARMISD::VQMOVNs, DL, HalfVT, DAG.getUNDEF(HalfVT),
17795 N0->getOperand(0), DAG.getConstant(0, DL, MVT::i32));
17796 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
17797 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Bitcast,
17798 DAG.getValueType(ExtVT));
17799 }
17800
17801 auto IsUnsignedSaturate = [&](SDNode *Min) {
17802 // For unsigned, we just need to check for <= 0xffff
17803 if (Min->getOpcode() != ISD::UMIN)
17804 return false;
17805
17806 APInt SaturateC;
17807 if (VT == MVT::v4i32)
17808 SaturateC = APInt(32, (1 << 16) - 1, true);
17809 else //if (VT == MVT::v8i16)
17810 SaturateC = APInt(16, (1 << 8) - 1, true);
17811
17812 APInt MinC;
17813 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) ||
17814 MinC != SaturateC)
17815 return false;
17816 return true;
17817 };
17818
17819 if (IsUnsignedSaturate(N)) {
17820 SDLoc DL(N);
17821 MVT HalfVT;
17822 unsigned ExtConst;
17823 if (VT == MVT::v4i32) {
17824 HalfVT = MVT::v8i16;
17825 ExtConst = 0x0000FFFF;
17826 } else { //if (VT == MVT::v8i16)
17827 HalfVT = MVT::v16i8;
17828 ExtConst = 0x00FF;
17829 }
17830
17831 // Create a VQMOVNB with undef top lanes, then ZExt into the top half with
17832 // an AND. That extend will hopefully be removed if only the bottom bits are
17833 // demanded (though a truncating store, for example).
17834 SDValue VQMOVN =
17835 DAG.getNode(ARMISD::VQMOVNu, DL, HalfVT, DAG.getUNDEF(HalfVT), N0,
17836 DAG.getConstant(0, DL, MVT::i32));
17837 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
17838 return DAG.getNode(ISD::AND, DL, VT, Bitcast,
17839 DAG.getConstant(ExtConst, DL, VT));
17840 }
17841
17842 return SDValue();
17843 }
17844
isPowerOf2Constant(SDValue V)17845 static const APInt *isPowerOf2Constant(SDValue V) {
17846 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
17847 if (!C)
17848 return nullptr;
17849 const APInt *CV = &C->getAPIntValue();
17850 return CV->isPowerOf2() ? CV : nullptr;
17851 }
17852
PerformCMOVToBFICombine(SDNode * CMOV,SelectionDAG & DAG) const17853 SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
17854 // If we have a CMOV, OR and AND combination such as:
17855 // if (x & CN)
17856 // y |= CM;
17857 //
17858 // And:
17859 // * CN is a single bit;
17860 // * All bits covered by CM are known zero in y
17861 //
17862 // Then we can convert this into a sequence of BFI instructions. This will
17863 // always be a win if CM is a single bit, will always be no worse than the
17864 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
17865 // three bits (due to the extra IT instruction).
17866
17867 SDValue Op0 = CMOV->getOperand(0);
17868 SDValue Op1 = CMOV->getOperand(1);
17869 auto CCNode = cast<ConstantSDNode>(CMOV->getOperand(2));
17870 auto CC = CCNode->getAPIntValue().getLimitedValue();
17871 SDValue CmpZ = CMOV->getOperand(4);
17872
17873 // The compare must be against zero.
17874 if (!isNullConstant(CmpZ->getOperand(1)))
17875 return SDValue();
17876
17877 assert(CmpZ->getOpcode() == ARMISD::CMPZ);
17878 SDValue And = CmpZ->getOperand(0);
17879 if (And->getOpcode() != ISD::AND)
17880 return SDValue();
17881 const APInt *AndC = isPowerOf2Constant(And->getOperand(1));
17882 if (!AndC)
17883 return SDValue();
17884 SDValue X = And->getOperand(0);
17885
17886 if (CC == ARMCC::EQ) {
17887 // We're performing an "equal to zero" compare. Swap the operands so we
17888 // canonicalize on a "not equal to zero" compare.
17889 std::swap(Op0, Op1);
17890 } else {
17891 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?");
17892 }
17893
17894 if (Op1->getOpcode() != ISD::OR)
17895 return SDValue();
17896
17897 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Op1->getOperand(1));
17898 if (!OrC)
17899 return SDValue();
17900 SDValue Y = Op1->getOperand(0);
17901
17902 if (Op0 != Y)
17903 return SDValue();
17904
17905 // Now, is it profitable to continue?
17906 APInt OrCI = OrC->getAPIntValue();
17907 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
17908 if (OrCI.countPopulation() > Heuristic)
17909 return SDValue();
17910
17911 // Lastly, can we determine that the bits defined by OrCI
17912 // are zero in Y?
17913 KnownBits Known = DAG.computeKnownBits(Y);
17914 if ((OrCI & Known.Zero) != OrCI)
17915 return SDValue();
17916
17917 // OK, we can do the combine.
17918 SDValue V = Y;
17919 SDLoc dl(X);
17920 EVT VT = X.getValueType();
17921 unsigned BitInX = AndC->logBase2();
17922
17923 if (BitInX != 0) {
17924 // We must shift X first.
17925 X = DAG.getNode(ISD::SRL, dl, VT, X,
17926 DAG.getConstant(BitInX, dl, VT));
17927 }
17928
17929 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
17930 BitInY < NumActiveBits; ++BitInY) {
17931 if (OrCI[BitInY] == 0)
17932 continue;
17933 APInt Mask(VT.getSizeInBits(), 0);
17934 Mask.setBit(BitInY);
17935 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X,
17936 // Confusingly, the operand is an *inverted* mask.
17937 DAG.getConstant(~Mask, dl, VT));
17938 }
17939
17940 return V;
17941 }
17942
17943 // Given N, the value controlling the conditional branch, search for the loop
17944 // intrinsic, returning it, along with how the value is used. We need to handle
17945 // patterns such as the following:
17946 // (brcond (xor (setcc (loop.decrement), 0, ne), 1), exit)
17947 // (brcond (setcc (loop.decrement), 0, eq), exit)
17948 // (brcond (setcc (loop.decrement), 0, ne), header)
SearchLoopIntrinsic(SDValue N,ISD::CondCode & CC,int & Imm,bool & Negate)17949 static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
17950 bool &Negate) {
17951 switch (N->getOpcode()) {
17952 default:
17953 break;
17954 case ISD::XOR: {
17955 if (!isa<ConstantSDNode>(N.getOperand(1)))
17956 return SDValue();
17957 if (!cast<ConstantSDNode>(N.getOperand(1))->isOne())
17958 return SDValue();
17959 Negate = !Negate;
17960 return SearchLoopIntrinsic(N.getOperand(0), CC, Imm, Negate);
17961 }
17962 case ISD::SETCC: {
17963 auto *Const = dyn_cast<ConstantSDNode>(N.getOperand(1));
17964 if (!Const)
17965 return SDValue();
17966 if (Const->isZero())
17967 Imm = 0;
17968 else if (Const->isOne())
17969 Imm = 1;
17970 else
17971 return SDValue();
17972 CC = cast<CondCodeSDNode>(N.getOperand(2))->get();
17973 return SearchLoopIntrinsic(N->getOperand(0), CC, Imm, Negate);
17974 }
17975 case ISD::INTRINSIC_W_CHAIN: {
17976 unsigned IntOp = cast<ConstantSDNode>(N.getOperand(1))->getZExtValue();
17977 if (IntOp != Intrinsic::test_start_loop_iterations &&
17978 IntOp != Intrinsic::loop_decrement_reg)
17979 return SDValue();
17980 return N;
17981 }
17982 }
17983 return SDValue();
17984 }
17985
PerformHWLoopCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * ST)17986 static SDValue PerformHWLoopCombine(SDNode *N,
17987 TargetLowering::DAGCombinerInfo &DCI,
17988 const ARMSubtarget *ST) {
17989
17990 // The hwloop intrinsics that we're interested are used for control-flow,
17991 // either for entering or exiting the loop:
17992 // - test.start.loop.iterations will test whether its operand is zero. If it
17993 // is zero, the proceeding branch should not enter the loop.
17994 // - loop.decrement.reg also tests whether its operand is zero. If it is
17995 // zero, the proceeding branch should not branch back to the beginning of
17996 // the loop.
17997 // So here, we need to check that how the brcond is using the result of each
17998 // of the intrinsics to ensure that we're branching to the right place at the
17999 // right time.
18000
18001 ISD::CondCode CC;
18002 SDValue Cond;
18003 int Imm = 1;
18004 bool Negate = false;
18005 SDValue Chain = N->getOperand(0);
18006 SDValue Dest;
18007
18008 if (N->getOpcode() == ISD::BRCOND) {
18009 CC = ISD::SETEQ;
18010 Cond = N->getOperand(1);
18011 Dest = N->getOperand(2);
18012 } else {
18013 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!");
18014 CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
18015 Cond = N->getOperand(2);
18016 Dest = N->getOperand(4);
18017 if (auto *Const = dyn_cast<ConstantSDNode>(N->getOperand(3))) {
18018 if (!Const->isOne() && !Const->isZero())
18019 return SDValue();
18020 Imm = Const->getZExtValue();
18021 } else
18022 return SDValue();
18023 }
18024
18025 SDValue Int = SearchLoopIntrinsic(Cond, CC, Imm, Negate);
18026 if (!Int)
18027 return SDValue();
18028
18029 if (Negate)
18030 CC = ISD::getSetCCInverse(CC, /* Integer inverse */ MVT::i32);
18031
18032 auto IsTrueIfZero = [](ISD::CondCode CC, int Imm) {
18033 return (CC == ISD::SETEQ && Imm == 0) ||
18034 (CC == ISD::SETNE && Imm == 1) ||
18035 (CC == ISD::SETLT && Imm == 1) ||
18036 (CC == ISD::SETULT && Imm == 1);
18037 };
18038
18039 auto IsFalseIfZero = [](ISD::CondCode CC, int Imm) {
18040 return (CC == ISD::SETEQ && Imm == 1) ||
18041 (CC == ISD::SETNE && Imm == 0) ||
18042 (CC == ISD::SETGT && Imm == 0) ||
18043 (CC == ISD::SETUGT && Imm == 0) ||
18044 (CC == ISD::SETGE && Imm == 1) ||
18045 (CC == ISD::SETUGE && Imm == 1);
18046 };
18047
18048 assert((IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) &&
18049 "unsupported condition");
18050
18051 SDLoc dl(Int);
18052 SelectionDAG &DAG = DCI.DAG;
18053 SDValue Elements = Int.getOperand(2);
18054 unsigned IntOp = cast<ConstantSDNode>(Int->getOperand(1))->getZExtValue();
18055 assert((N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BR)
18056 && "expected single br user");
18057 SDNode *Br = *N->use_begin();
18058 SDValue OtherTarget = Br->getOperand(1);
18059
18060 // Update the unconditional branch to branch to the given Dest.
18061 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
18062 SDValue NewBrOps[] = { Br->getOperand(0), Dest };
18063 SDValue NewBr = DAG.getNode(ISD::BR, SDLoc(Br), MVT::Other, NewBrOps);
18064 DAG.ReplaceAllUsesOfValueWith(SDValue(Br, 0), NewBr);
18065 };
18066
18067 if (IntOp == Intrinsic::test_start_loop_iterations) {
18068 SDValue Res;
18069 SDValue Setup = DAG.getNode(ARMISD::WLSSETUP, dl, MVT::i32, Elements);
18070 // We expect this 'instruction' to branch when the counter is zero.
18071 if (IsTrueIfZero(CC, Imm)) {
18072 SDValue Ops[] = {Chain, Setup, Dest};
18073 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
18074 } else {
18075 // The logic is the reverse of what we need for WLS, so find the other
18076 // basic block target: the target of the proceeding br.
18077 UpdateUncondBr(Br, Dest, DAG);
18078
18079 SDValue Ops[] = {Chain, Setup, OtherTarget};
18080 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
18081 }
18082 // Update LR count to the new value
18083 DAG.ReplaceAllUsesOfValueWith(Int.getValue(0), Setup);
18084 // Update chain
18085 DAG.ReplaceAllUsesOfValueWith(Int.getValue(2), Int.getOperand(0));
18086 return Res;
18087 } else {
18088 SDValue Size = DAG.getTargetConstant(
18089 cast<ConstantSDNode>(Int.getOperand(3))->getZExtValue(), dl, MVT::i32);
18090 SDValue Args[] = { Int.getOperand(0), Elements, Size, };
18091 SDValue LoopDec = DAG.getNode(ARMISD::LOOP_DEC, dl,
18092 DAG.getVTList(MVT::i32, MVT::Other), Args);
18093 DAG.ReplaceAllUsesWith(Int.getNode(), LoopDec.getNode());
18094
18095 // We expect this instruction to branch when the count is not zero.
18096 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
18097
18098 // Update the unconditional branch to target the loop preheader if we've
18099 // found the condition has been reversed.
18100 if (Target == OtherTarget)
18101 UpdateUncondBr(Br, Dest, DAG);
18102
18103 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
18104 SDValue(LoopDec.getNode(), 1), Chain);
18105
18106 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
18107 return DAG.getNode(ARMISD::LE, dl, MVT::Other, EndArgs);
18108 }
18109 return SDValue();
18110 }
18111
18112 /// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
18113 SDValue
PerformBRCONDCombine(SDNode * N,SelectionDAG & DAG) const18114 ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
18115 SDValue Cmp = N->getOperand(4);
18116 if (Cmp.getOpcode() != ARMISD::CMPZ)
18117 // Only looking at NE cases.
18118 return SDValue();
18119
18120 EVT VT = N->getValueType(0);
18121 SDLoc dl(N);
18122 SDValue LHS = Cmp.getOperand(0);
18123 SDValue RHS = Cmp.getOperand(1);
18124 SDValue Chain = N->getOperand(0);
18125 SDValue BB = N->getOperand(1);
18126 SDValue ARMcc = N->getOperand(2);
18127 ARMCC::CondCodes CC =
18128 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
18129
18130 // (brcond Chain BB ne CPSR (cmpz (and (cmov 0 1 CC CPSR Cmp) 1) 0))
18131 // -> (brcond Chain BB CC CPSR Cmp)
18132 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
18133 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV &&
18134 LHS->getOperand(0)->hasOneUse()) {
18135 auto *LHS00C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(0));
18136 auto *LHS01C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(1));
18137 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
18138 auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
18139 if ((LHS00C && LHS00C->getZExtValue() == 0) &&
18140 (LHS01C && LHS01C->getZExtValue() == 1) &&
18141 (LHS1C && LHS1C->getZExtValue() == 1) &&
18142 (RHSC && RHSC->getZExtValue() == 0)) {
18143 return DAG.getNode(
18144 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2),
18145 LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4));
18146 }
18147 }
18148
18149 return SDValue();
18150 }
18151
18152 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
18153 SDValue
PerformCMOVCombine(SDNode * N,SelectionDAG & DAG) const18154 ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
18155 SDValue Cmp = N->getOperand(4);
18156 if (Cmp.getOpcode() != ARMISD::CMPZ)
18157 // Only looking at EQ and NE cases.
18158 return SDValue();
18159
18160 EVT VT = N->getValueType(0);
18161 SDLoc dl(N);
18162 SDValue LHS = Cmp.getOperand(0);
18163 SDValue RHS = Cmp.getOperand(1);
18164 SDValue FalseVal = N->getOperand(0);
18165 SDValue TrueVal = N->getOperand(1);
18166 SDValue ARMcc = N->getOperand(2);
18167 ARMCC::CondCodes CC =
18168 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
18169
18170 // BFI is only available on V6T2+.
18171 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
18172 SDValue R = PerformCMOVToBFICombine(N, DAG);
18173 if (R)
18174 return R;
18175 }
18176
18177 // Simplify
18178 // mov r1, r0
18179 // cmp r1, x
18180 // mov r0, y
18181 // moveq r0, x
18182 // to
18183 // cmp r0, x
18184 // movne r0, y
18185 //
18186 // mov r1, r0
18187 // cmp r1, x
18188 // mov r0, x
18189 // movne r0, y
18190 // to
18191 // cmp r0, x
18192 // movne r0, y
18193 /// FIXME: Turn this into a target neutral optimization?
18194 SDValue Res;
18195 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
18196 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc,
18197 N->getOperand(3), Cmp);
18198 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
18199 SDValue ARMcc;
18200 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
18201 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc,
18202 N->getOperand(3), NewCmp);
18203 }
18204
18205 // (cmov F T ne CPSR (cmpz (cmov 0 1 CC CPSR Cmp) 0))
18206 // -> (cmov F T CC CPSR Cmp)
18207 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse()) {
18208 auto *LHS0C = dyn_cast<ConstantSDNode>(LHS->getOperand(0));
18209 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
18210 auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
18211 if ((LHS0C && LHS0C->getZExtValue() == 0) &&
18212 (LHS1C && LHS1C->getZExtValue() == 1) &&
18213 (RHSC && RHSC->getZExtValue() == 0)) {
18214 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
18215 LHS->getOperand(2), LHS->getOperand(3),
18216 LHS->getOperand(4));
18217 }
18218 }
18219
18220 if (!VT.isInteger())
18221 return SDValue();
18222
18223 // Fold away an unneccessary CMPZ/CMOV
18224 // CMOV A, B, C1, $cpsr, (CMPZ (CMOV 1, 0, C2, D), 0) ->
18225 // if C1==EQ -> CMOV A, B, C2, $cpsr, D
18226 // if C1==NE -> CMOV A, B, NOT(C2), $cpsr, D
18227 if (N->getConstantOperandVal(2) == ARMCC::EQ ||
18228 N->getConstantOperandVal(2) == ARMCC::NE) {
18229 ARMCC::CondCodes Cond;
18230 if (SDValue C = IsCMPZCSINC(N->getOperand(4).getNode(), Cond)) {
18231 if (N->getConstantOperandVal(2) == ARMCC::NE)
18232 Cond = ARMCC::getOppositeCondition(Cond);
18233 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
18234 N->getOperand(1),
18235 DAG.getTargetConstant(Cond, SDLoc(N), MVT::i32),
18236 N->getOperand(3), C);
18237 }
18238 }
18239
18240 // Materialize a boolean comparison for integers so we can avoid branching.
18241 if (isNullConstant(FalseVal)) {
18242 if (CC == ARMCC::EQ && isOneConstant(TrueVal)) {
18243 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) {
18244 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it
18245 // right 5 bits will make that 32 be 1, otherwise it will be 0.
18246 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5
18247 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
18248 Res = DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Sub),
18249 DAG.getConstant(5, dl, MVT::i32));
18250 } else {
18251 // CMOV 0, 1, ==, (CMPZ x, y) ->
18252 // (ADDCARRY (SUB x, y), t:0, t:1)
18253 // where t = (SUBCARRY 0, (SUB x, y), 0)
18254 //
18255 // The SUBCARRY computes 0 - (x - y) and this will give a borrow when
18256 // x != y. In other words, a carry C == 1 when x == y, C == 0
18257 // otherwise.
18258 // The final ADDCARRY computes
18259 // x - y + (0 - (x - y)) + C == C
18260 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
18261 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
18262 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub);
18263 // ISD::SUBCARRY returns a borrow but we want the carry here
18264 // actually.
18265 SDValue Carry =
18266 DAG.getNode(ISD::SUB, dl, MVT::i32,
18267 DAG.getConstant(1, dl, MVT::i32), Neg.getValue(1));
18268 Res = DAG.getNode(ISD::ADDCARRY, dl, VTs, Sub, Neg, Carry);
18269 }
18270 } else if (CC == ARMCC::NE && !isNullConstant(RHS) &&
18271 (!Subtarget->isThumb1Only() || isPowerOf2Constant(TrueVal))) {
18272 // This seems pointless but will allow us to combine it further below.
18273 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBS x, y), z, !=, (SUBS x, y):1
18274 SDValue Sub =
18275 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
18276 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
18277 Sub.getValue(1), SDValue());
18278 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc,
18279 N->getOperand(3), CPSRGlue.getValue(1));
18280 FalseVal = Sub;
18281 }
18282 } else if (isNullConstant(TrueVal)) {
18283 if (CC == ARMCC::EQ && !isNullConstant(RHS) &&
18284 (!Subtarget->isThumb1Only() || isPowerOf2Constant(FalseVal))) {
18285 // This seems pointless but will allow us to combine it further below
18286 // Note that we change == for != as this is the dual for the case above.
18287 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBS x, y), z, !=, (SUBS x, y):1
18288 SDValue Sub =
18289 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
18290 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
18291 Sub.getValue(1), SDValue());
18292 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal,
18293 DAG.getConstant(ARMCC::NE, dl, MVT::i32),
18294 N->getOperand(3), CPSRGlue.getValue(1));
18295 FalseVal = Sub;
18296 }
18297 }
18298
18299 // On Thumb1, the DAG above may be further combined if z is a power of 2
18300 // (z == 2 ^ K).
18301 // CMOV (SUBS x, y), z, !=, (SUBS x, y):1 ->
18302 // t1 = (USUBO (SUB x, y), 1)
18303 // t2 = (SUBCARRY (SUB x, y), t1:0, t1:1)
18304 // Result = if K != 0 then (SHL t2:0, K) else t2:0
18305 //
18306 // This also handles the special case of comparing against zero; it's
18307 // essentially, the same pattern, except there's no SUBS:
18308 // CMOV x, z, !=, (CMPZ x, 0) ->
18309 // t1 = (USUBO x, 1)
18310 // t2 = (SUBCARRY x, t1:0, t1:1)
18311 // Result = if K != 0 then (SHL t2:0, K) else t2:0
18312 const APInt *TrueConst;
18313 if (Subtarget->isThumb1Only() && CC == ARMCC::NE &&
18314 ((FalseVal.getOpcode() == ARMISD::SUBS &&
18315 FalseVal.getOperand(0) == LHS && FalseVal.getOperand(1) == RHS) ||
18316 (FalseVal == LHS && isNullConstant(RHS))) &&
18317 (TrueConst = isPowerOf2Constant(TrueVal))) {
18318 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
18319 unsigned ShiftAmount = TrueConst->logBase2();
18320 if (ShiftAmount)
18321 TrueVal = DAG.getConstant(1, dl, VT);
18322 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal);
18323 Res = DAG.getNode(ISD::SUBCARRY, dl, VTs, FalseVal, Subc, Subc.getValue(1));
18324
18325 if (ShiftAmount)
18326 Res = DAG.getNode(ISD::SHL, dl, VT, Res,
18327 DAG.getConstant(ShiftAmount, dl, MVT::i32));
18328 }
18329
18330 if (Res.getNode()) {
18331 KnownBits Known = DAG.computeKnownBits(SDValue(N,0));
18332 // Capture demanded bits information that would be otherwise lost.
18333 if (Known.Zero == 0xfffffffe)
18334 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
18335 DAG.getValueType(MVT::i1));
18336 else if (Known.Zero == 0xffffff00)
18337 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
18338 DAG.getValueType(MVT::i8));
18339 else if (Known.Zero == 0xffff0000)
18340 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
18341 DAG.getValueType(MVT::i16));
18342 }
18343
18344 return Res;
18345 }
18346
PerformBITCASTCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * ST)18347 static SDValue PerformBITCASTCombine(SDNode *N,
18348 TargetLowering::DAGCombinerInfo &DCI,
18349 const ARMSubtarget *ST) {
18350 SelectionDAG &DAG = DCI.DAG;
18351 SDValue Src = N->getOperand(0);
18352 EVT DstVT = N->getValueType(0);
18353
18354 // Convert v4f32 bitcast (v4i32 vdup (i32)) -> v4f32 vdup (i32) under MVE.
18355 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) {
18356 EVT SrcVT = Src.getValueType();
18357 if (SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits())
18358 return DAG.getNode(ARMISD::VDUP, SDLoc(N), DstVT, Src.getOperand(0));
18359 }
18360
18361 // We may have a bitcast of something that has already had this bitcast
18362 // combine performed on it, so skip past any VECTOR_REG_CASTs.
18363 while (Src.getOpcode() == ARMISD::VECTOR_REG_CAST)
18364 Src = Src.getOperand(0);
18365
18366 // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that
18367 // would be generated is at least the width of the element type.
18368 EVT SrcVT = Src.getValueType();
18369 if ((Src.getOpcode() == ARMISD::VMOVIMM ||
18370 Src.getOpcode() == ARMISD::VMVNIMM ||
18371 Src.getOpcode() == ARMISD::VMOVFPIMM) &&
18372 SrcVT.getScalarSizeInBits() <= DstVT.getScalarSizeInBits() &&
18373 DAG.getDataLayout().isBigEndian())
18374 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(N), DstVT, Src);
18375
18376 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x
18377 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
18378 return R;
18379
18380 return SDValue();
18381 }
18382
18383 // Some combines for the MVETrunc truncations legalizer helper. Also lowers the
18384 // node into stack operations after legalizeOps.
PerformMVETruncCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI) const18385 SDValue ARMTargetLowering::PerformMVETruncCombine(
18386 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
18387 SelectionDAG &DAG = DCI.DAG;
18388 EVT VT = N->getValueType(0);
18389 SDLoc DL(N);
18390
18391 // MVETrunc(Undef, Undef) -> Undef
18392 if (all_of(N->ops(), [](SDValue Op) { return Op.isUndef(); }))
18393 return DAG.getUNDEF(VT);
18394
18395 // MVETrunc(MVETrunc a b, MVETrunc c, d) -> MVETrunc
18396 if (N->getNumOperands() == 2 &&
18397 N->getOperand(0).getOpcode() == ARMISD::MVETRUNC &&
18398 N->getOperand(1).getOpcode() == ARMISD::MVETRUNC)
18399 return DAG.getNode(ARMISD::MVETRUNC, DL, VT, N->getOperand(0).getOperand(0),
18400 N->getOperand(0).getOperand(1),
18401 N->getOperand(1).getOperand(0),
18402 N->getOperand(1).getOperand(1));
18403
18404 // MVETrunc(shuffle, shuffle) -> VMOVN
18405 if (N->getNumOperands() == 2 &&
18406 N->getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE &&
18407 N->getOperand(1).getOpcode() == ISD::VECTOR_SHUFFLE) {
18408 auto *S0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode());
18409 auto *S1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode());
18410
18411 if (S0->getOperand(0) == S1->getOperand(0) &&
18412 S0->getOperand(1) == S1->getOperand(1)) {
18413 // Construct complete shuffle mask
18414 SmallVector<int, 8> Mask(S0->getMask());
18415 Mask.append(S1->getMask().begin(), S1->getMask().end());
18416
18417 if (isVMOVNTruncMask(Mask, VT, false))
18418 return DAG.getNode(
18419 ARMISD::VMOVN, DL, VT,
18420 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)),
18421 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)),
18422 DAG.getConstant(1, DL, MVT::i32));
18423 if (isVMOVNTruncMask(Mask, VT, true))
18424 return DAG.getNode(
18425 ARMISD::VMOVN, DL, VT,
18426 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)),
18427 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)),
18428 DAG.getConstant(1, DL, MVT::i32));
18429 }
18430 }
18431
18432 // For MVETrunc of a buildvector or shuffle, it can be beneficial to lower the
18433 // truncate to a buildvector to allow the generic optimisations to kick in.
18434 if (all_of(N->ops(), [](SDValue Op) {
18435 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18436 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18437 (Op.getOpcode() == ISD::BITCAST &&
18438 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18439 })) {
18440 SmallVector<SDValue, 8> Extracts;
18441 for (unsigned Op = 0; Op < N->getNumOperands(); Op++) {
18442 SDValue O = N->getOperand(Op);
18443 for (unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18444 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, O,
18445 DAG.getConstant(i, DL, MVT::i32));
18446 Extracts.push_back(Ext);
18447 }
18448 }
18449 return DAG.getBuildVector(VT, DL, Extracts);
18450 }
18451
18452 // If we are late in the legalization process and nothing has optimised
18453 // the trunc to anything better, lower it to a stack store and reload,
18454 // performing the truncation whilst keeping the lanes in the correct order:
18455 // VSTRH.32 a, stack; VSTRH.32 b, stack+8; VLDRW.32 stack;
18456 if (!DCI.isAfterLegalizeDAG())
18457 return SDValue();
18458
18459 SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::Fixed(16), Align(4));
18460 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18461 int NumIns = N->getNumOperands();
18462 assert((NumIns == 2 || NumIns == 4) &&
18463 "Expected 2 or 4 inputs to an MVETrunc");
18464 EVT StoreVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
18465 if (N->getNumOperands() == 4)
18466 StoreVT = StoreVT.getHalfNumVectorElementsVT(*DAG.getContext());
18467
18468 SmallVector<SDValue> Chains;
18469 for (int I = 0; I < NumIns; I++) {
18470 SDValue Ptr = DAG.getNode(
18471 ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
18472 DAG.getConstant(I * 16 / NumIns, DL, StackPtr.getValueType()));
18473 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
18474 DAG.getMachineFunction(), SPFI, I * 16 / NumIns);
18475 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), DL, N->getOperand(I),
18476 Ptr, MPI, StoreVT, Align(4));
18477 Chains.push_back(Ch);
18478 }
18479
18480 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
18481 MachinePointerInfo MPI =
18482 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI, 0);
18483 return DAG.getLoad(VT, DL, Chain, StackPtr, MPI, Align(4));
18484 }
18485
18486 // Take a MVEEXT(load x) and split that into (extload x, extload x+8)
PerformSplittingMVEEXTToWideningLoad(SDNode * N,SelectionDAG & DAG)18487 static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N,
18488 SelectionDAG &DAG) {
18489 SDValue N0 = N->getOperand(0);
18490 LoadSDNode *LD = dyn_cast<LoadSDNode>(N0.getNode());
18491 if (!LD || !LD->isSimple() || !N0.hasOneUse() || LD->isIndexed())
18492 return SDValue();
18493
18494 EVT FromVT = LD->getMemoryVT();
18495 EVT ToVT = N->getValueType(0);
18496 if (!ToVT.isVector())
18497 return SDValue();
18498 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements() * 2);
18499 EVT ToEltVT = ToVT.getVectorElementType();
18500 EVT FromEltVT = FromVT.getVectorElementType();
18501
18502 unsigned NumElements = 0;
18503 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18504 NumElements = 4;
18505 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18506 NumElements = 8;
18507 assert(NumElements != 0);
18508
18509 ISD::LoadExtType NewExtType =
18510 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
18511 if (LD->getExtensionType() != ISD::NON_EXTLOAD &&
18512 LD->getExtensionType() != ISD::EXTLOAD &&
18513 LD->getExtensionType() != NewExtType)
18514 return SDValue();
18515
18516 LLVMContext &C = *DAG.getContext();
18517 SDLoc DL(LD);
18518 // Details about the old load
18519 SDValue Ch = LD->getChain();
18520 SDValue BasePtr = LD->getBasePtr();
18521 Align Alignment = LD->getOriginalAlign();
18522 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
18523 AAMDNodes AAInfo = LD->getAAInfo();
18524
18525 SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
18526 EVT NewFromVT = EVT::getVectorVT(
18527 C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
18528 EVT NewToVT = EVT::getVectorVT(
18529 C, EVT::getIntegerVT(C, ToEltVT.getScalarSizeInBits()), NumElements);
18530
18531 SmallVector<SDValue, 4> Loads;
18532 SmallVector<SDValue, 4> Chains;
18533 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
18534 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
18535 SDValue NewPtr =
18536 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset));
18537
18538 SDValue NewLoad =
18539 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
18540 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18541 Alignment, MMOFlags, AAInfo);
18542 Loads.push_back(NewLoad);
18543 Chains.push_back(SDValue(NewLoad.getNode(), 1));
18544 }
18545
18546 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
18547 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
18548 return DAG.getMergeValues(Loads, DL);
18549 }
18550
18551 // Perform combines for MVEEXT. If it has not be optimized to anything better
18552 // before lowering, it gets converted to stack store and extloads performing the
18553 // extend whilst still keeping the same lane ordering.
PerformMVEExtCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI) const18554 SDValue ARMTargetLowering::PerformMVEExtCombine(
18555 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
18556 SelectionDAG &DAG = DCI.DAG;
18557 EVT VT = N->getValueType(0);
18558 SDLoc DL(N);
18559 assert(N->getNumValues() == 2 && "Expected MVEEXT with 2 elements");
18560 assert((VT == MVT::v4i32 || VT == MVT::v8i16) && "Unexpected MVEEXT type");
18561
18562 EVT ExtVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18563 *DAG.getContext());
18564 auto Extend = [&](SDValue V) {
18565 SDValue VVT = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, V);
18566 return N->getOpcode() == ARMISD::MVESEXT
18567 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, VVT,
18568 DAG.getValueType(ExtVT))
18569 : DAG.getZeroExtendInReg(VVT, DL, ExtVT);
18570 };
18571
18572 // MVEEXT(VDUP) -> SIGN_EXTEND_INREG(VDUP)
18573 if (N->getOperand(0).getOpcode() == ARMISD::VDUP) {
18574 SDValue Ext = Extend(N->getOperand(0));
18575 return DAG.getMergeValues({Ext, Ext}, DL);
18576 }
18577
18578 // MVEEXT(shuffle) -> SIGN_EXTEND_INREG/ZERO_EXTEND_INREG
18579 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N->getOperand(0))) {
18580 ArrayRef<int> Mask = SVN->getMask();
18581 assert(Mask.size() == 2 * VT.getVectorNumElements());
18582 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18583 unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16;
18584 SDValue Op0 = SVN->getOperand(0);
18585 SDValue Op1 = SVN->getOperand(1);
18586
18587 auto CheckInregMask = [&](int Start, int Offset) {
18588 for (int Idx = 0, E = VT.getVectorNumElements(); Idx < E; ++Idx)
18589 if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 + Offset)
18590 return false;
18591 return true;
18592 };
18593 SDValue V0 = SDValue(N, 0);
18594 SDValue V1 = SDValue(N, 1);
18595 if (CheckInregMask(0, 0))
18596 V0 = Extend(Op0);
18597 else if (CheckInregMask(0, 1))
18598 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0));
18599 else if (CheckInregMask(0, Mask.size()))
18600 V0 = Extend(Op1);
18601 else if (CheckInregMask(0, Mask.size() + 1))
18602 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1));
18603
18604 if (CheckInregMask(VT.getVectorNumElements(), Mask.size()))
18605 V1 = Extend(Op1);
18606 else if (CheckInregMask(VT.getVectorNumElements(), Mask.size() + 1))
18607 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1));
18608 else if (CheckInregMask(VT.getVectorNumElements(), 0))
18609 V1 = Extend(Op0);
18610 else if (CheckInregMask(VT.getVectorNumElements(), 1))
18611 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0));
18612
18613 if (V0.getNode() != N || V1.getNode() != N)
18614 return DAG.getMergeValues({V0, V1}, DL);
18615 }
18616
18617 // MVEEXT(load) -> extload, extload
18618 if (N->getOperand(0)->getOpcode() == ISD::LOAD)
18619 if (SDValue L = PerformSplittingMVEEXTToWideningLoad(N, DAG))
18620 return L;
18621
18622 if (!DCI.isAfterLegalizeDAG())
18623 return SDValue();
18624
18625 // Lower to a stack store and reload:
18626 // VSTRW.32 a, stack; VLDRH.32 stack; VLDRH.32 stack+8;
18627 SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::Fixed(16), Align(4));
18628 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18629 int NumOuts = N->getNumValues();
18630 assert((NumOuts == 2 || NumOuts == 4) &&
18631 "Expected 2 or 4 outputs to an MVEEXT");
18632 EVT LoadVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18633 *DAG.getContext());
18634 if (N->getNumOperands() == 4)
18635 LoadVT = LoadVT.getHalfNumVectorElementsVT(*DAG.getContext());
18636
18637 MachinePointerInfo MPI =
18638 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI, 0);
18639 SDValue Chain = DAG.getStore(DAG.getEntryNode(), DL, N->getOperand(0),
18640 StackPtr, MPI, Align(4));
18641
18642 SmallVector<SDValue> Loads;
18643 for (int I = 0; I < NumOuts; I++) {
18644 SDValue Ptr = DAG.getNode(
18645 ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
18646 DAG.getConstant(I * 16 / NumOuts, DL, StackPtr.getValueType()));
18647 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
18648 DAG.getMachineFunction(), SPFI, I * 16 / NumOuts);
18649 SDValue Load = DAG.getExtLoad(
18650 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, DL,
18651 VT, Chain, Ptr, MPI, LoadVT, Align(4));
18652 Loads.push_back(Load);
18653 }
18654
18655 return DAG.getMergeValues(Loads, DL);
18656 }
18657
PerformDAGCombine(SDNode * N,DAGCombinerInfo & DCI) const18658 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
18659 DAGCombinerInfo &DCI) const {
18660 switch (N->getOpcode()) {
18661 default: break;
18662 case ISD::SELECT_CC:
18663 case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget);
18664 case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget);
18665 case ISD::SETCC: return PerformVSetCCToVCTPCombine(N, DCI, Subtarget);
18666 case ISD::ABS: return PerformABSCombine(N, DCI, Subtarget);
18667 case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget);
18668 case ARMISD::UMLAL: return PerformUMLALCombine(N, DCI.DAG, Subtarget);
18669 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
18670 case ISD::SUB: return PerformSUBCombine(N, DCI, Subtarget);
18671 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
18672 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
18673 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
18674 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
18675 case ISD::BRCOND:
18676 case ISD::BR_CC: return PerformHWLoopCombine(N, DCI, Subtarget);
18677 case ARMISD::ADDC:
18678 case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget);
18679 case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget);
18680 case ARMISD::BFI: return PerformBFICombine(N, DCI.DAG);
18681 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
18682 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
18683 case ARMISD::VMOVhr: return PerformVMOVhrCombine(N, DCI);
18684 case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DCI.DAG);
18685 case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget);
18686 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
18687 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
18688 case ISD::EXTRACT_VECTOR_ELT:
18689 return PerformExtractEltCombine(N, DCI, Subtarget);
18690 case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DCI.DAG);
18691 case ISD::INSERT_SUBVECTOR: return PerformInsertSubvectorCombine(N, DCI);
18692 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
18693 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI, Subtarget);
18694 case ARMISD::VDUP: return PerformVDUPCombine(N, DCI.DAG, Subtarget);
18695 case ISD::FP_TO_SINT:
18696 case ISD::FP_TO_UINT:
18697 return PerformVCVTCombine(N, DCI.DAG, Subtarget);
18698 case ISD::FADD:
18699 return PerformFAddVSelectCombine(N, DCI.DAG, Subtarget);
18700 case ISD::FDIV:
18701 return PerformVDIVCombine(N, DCI.DAG, Subtarget);
18702 case ISD::INTRINSIC_WO_CHAIN:
18703 return PerformIntrinsicCombine(N, DCI);
18704 case ISD::SHL:
18705 case ISD::SRA:
18706 case ISD::SRL:
18707 return PerformShiftCombine(N, DCI, Subtarget);
18708 case ISD::SIGN_EXTEND:
18709 case ISD::ZERO_EXTEND:
18710 case ISD::ANY_EXTEND:
18711 return PerformExtendCombine(N, DCI.DAG, Subtarget);
18712 case ISD::FP_EXTEND:
18713 return PerformFPExtendCombine(N, DCI.DAG, Subtarget);
18714 case ISD::SMIN:
18715 case ISD::UMIN:
18716 case ISD::SMAX:
18717 case ISD::UMAX:
18718 return PerformMinMaxCombine(N, DCI.DAG, Subtarget);
18719 case ARMISD::CMOV:
18720 return PerformCMOVCombine(N, DCI.DAG);
18721 case ARMISD::BRCOND:
18722 return PerformBRCONDCombine(N, DCI.DAG);
18723 case ARMISD::CMPZ:
18724 return PerformCMPZCombine(N, DCI.DAG);
18725 case ARMISD::CSINC:
18726 case ARMISD::CSINV:
18727 case ARMISD::CSNEG:
18728 return PerformCSETCombine(N, DCI.DAG);
18729 case ISD::LOAD:
18730 return PerformLOADCombine(N, DCI, Subtarget);
18731 case ARMISD::VLD1DUP:
18732 case ARMISD::VLD2DUP:
18733 case ARMISD::VLD3DUP:
18734 case ARMISD::VLD4DUP:
18735 return PerformVLDCombine(N, DCI);
18736 case ARMISD::BUILD_VECTOR:
18737 return PerformARMBUILD_VECTORCombine(N, DCI);
18738 case ISD::BITCAST:
18739 return PerformBITCASTCombine(N, DCI, Subtarget);
18740 case ARMISD::PREDICATE_CAST:
18741 return PerformPREDICATE_CASTCombine(N, DCI);
18742 case ARMISD::VECTOR_REG_CAST:
18743 return PerformVECTOR_REG_CASTCombine(N, DCI.DAG, Subtarget);
18744 case ARMISD::MVETRUNC:
18745 return PerformMVETruncCombine(N, DCI);
18746 case ARMISD::MVESEXT:
18747 case ARMISD::MVEZEXT:
18748 return PerformMVEExtCombine(N, DCI);
18749 case ARMISD::VCMP:
18750 return PerformVCMPCombine(N, DCI.DAG, Subtarget);
18751 case ISD::VECREDUCE_ADD:
18752 return PerformVECREDUCE_ADDCombine(N, DCI.DAG, Subtarget);
18753 case ARMISD::VMOVN:
18754 return PerformVMOVNCombine(N, DCI);
18755 case ARMISD::VQMOVNs:
18756 case ARMISD::VQMOVNu:
18757 return PerformVQMOVNCombine(N, DCI);
18758 case ARMISD::ASRL:
18759 case ARMISD::LSRL:
18760 case ARMISD::LSLL:
18761 return PerformLongShiftCombine(N, DCI.DAG);
18762 case ARMISD::SMULWB: {
18763 unsigned BitWidth = N->getValueType(0).getSizeInBits();
18764 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
18765 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
18766 return SDValue();
18767 break;
18768 }
18769 case ARMISD::SMULWT: {
18770 unsigned BitWidth = N->getValueType(0).getSizeInBits();
18771 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16);
18772 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
18773 return SDValue();
18774 break;
18775 }
18776 case ARMISD::SMLALBB:
18777 case ARMISD::QADD16b:
18778 case ARMISD::QSUB16b:
18779 case ARMISD::UQADD16b:
18780 case ARMISD::UQSUB16b: {
18781 unsigned BitWidth = N->getValueType(0).getSizeInBits();
18782 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
18783 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
18784 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
18785 return SDValue();
18786 break;
18787 }
18788 case ARMISD::SMLALBT: {
18789 unsigned LowWidth = N->getOperand(0).getValueType().getSizeInBits();
18790 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16);
18791 unsigned HighWidth = N->getOperand(1).getValueType().getSizeInBits();
18792 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16);
18793 if ((SimplifyDemandedBits(N->getOperand(0), LowMask, DCI)) ||
18794 (SimplifyDemandedBits(N->getOperand(1), HighMask, DCI)))
18795 return SDValue();
18796 break;
18797 }
18798 case ARMISD::SMLALTB: {
18799 unsigned HighWidth = N->getOperand(0).getValueType().getSizeInBits();
18800 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16);
18801 unsigned LowWidth = N->getOperand(1).getValueType().getSizeInBits();
18802 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16);
18803 if ((SimplifyDemandedBits(N->getOperand(0), HighMask, DCI)) ||
18804 (SimplifyDemandedBits(N->getOperand(1), LowMask, DCI)))
18805 return SDValue();
18806 break;
18807 }
18808 case ARMISD::SMLALTT: {
18809 unsigned BitWidth = N->getValueType(0).getSizeInBits();
18810 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16);
18811 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
18812 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
18813 return SDValue();
18814 break;
18815 }
18816 case ARMISD::QADD8b:
18817 case ARMISD::QSUB8b:
18818 case ARMISD::UQADD8b:
18819 case ARMISD::UQSUB8b: {
18820 unsigned BitWidth = N->getValueType(0).getSizeInBits();
18821 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 8);
18822 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
18823 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
18824 return SDValue();
18825 break;
18826 }
18827 case ISD::INTRINSIC_VOID:
18828 case ISD::INTRINSIC_W_CHAIN:
18829 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
18830 case Intrinsic::arm_neon_vld1:
18831 case Intrinsic::arm_neon_vld1x2:
18832 case Intrinsic::arm_neon_vld1x3:
18833 case Intrinsic::arm_neon_vld1x4:
18834 case Intrinsic::arm_neon_vld2:
18835 case Intrinsic::arm_neon_vld3:
18836 case Intrinsic::arm_neon_vld4:
18837 case Intrinsic::arm_neon_vld2lane:
18838 case Intrinsic::arm_neon_vld3lane:
18839 case Intrinsic::arm_neon_vld4lane:
18840 case Intrinsic::arm_neon_vld2dup:
18841 case Intrinsic::arm_neon_vld3dup:
18842 case Intrinsic::arm_neon_vld4dup:
18843 case Intrinsic::arm_neon_vst1:
18844 case Intrinsic::arm_neon_vst1x2:
18845 case Intrinsic::arm_neon_vst1x3:
18846 case Intrinsic::arm_neon_vst1x4:
18847 case Intrinsic::arm_neon_vst2:
18848 case Intrinsic::arm_neon_vst3:
18849 case Intrinsic::arm_neon_vst4:
18850 case Intrinsic::arm_neon_vst2lane:
18851 case Intrinsic::arm_neon_vst3lane:
18852 case Intrinsic::arm_neon_vst4lane:
18853 return PerformVLDCombine(N, DCI);
18854 case Intrinsic::arm_mve_vld2q:
18855 case Intrinsic::arm_mve_vld4q:
18856 case Intrinsic::arm_mve_vst2q:
18857 case Intrinsic::arm_mve_vst4q:
18858 return PerformMVEVLDCombine(N, DCI);
18859 default: break;
18860 }
18861 break;
18862 }
18863 return SDValue();
18864 }
18865
isDesirableToTransformToIntegerOp(unsigned Opc,EVT VT) const18866 bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
18867 EVT VT) const {
18868 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
18869 }
18870
allowsMisalignedMemoryAccesses(EVT VT,unsigned,Align Alignment,MachineMemOperand::Flags,unsigned * Fast) const18871 bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned,
18872 Align Alignment,
18873 MachineMemOperand::Flags,
18874 unsigned *Fast) const {
18875 // Depends what it gets converted into if the type is weird.
18876 if (!VT.isSimple())
18877 return false;
18878
18879 // The AllowsUnaligned flag models the SCTLR.A setting in ARM cpus
18880 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
18881 auto Ty = VT.getSimpleVT().SimpleTy;
18882
18883 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
18884 // Unaligned access can use (for example) LRDB, LRDH, LDR
18885 if (AllowsUnaligned) {
18886 if (Fast)
18887 *Fast = Subtarget->hasV7Ops();
18888 return true;
18889 }
18890 }
18891
18892 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
18893 // For any little-endian targets with neon, we can support unaligned ld/st
18894 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
18895 // A big-endian target may also explicitly support unaligned accesses
18896 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
18897 if (Fast)
18898 *Fast = 1;
18899 return true;
18900 }
18901 }
18902
18903 if (!Subtarget->hasMVEIntegerOps())
18904 return false;
18905
18906 // These are for predicates
18907 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
18908 Ty == MVT::v2i1)) {
18909 if (Fast)
18910 *Fast = 1;
18911 return true;
18912 }
18913
18914 // These are for truncated stores/narrowing loads. They are fine so long as
18915 // the alignment is at least the size of the item being loaded
18916 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
18917 Alignment >= VT.getScalarSizeInBits() / 8) {
18918 if (Fast)
18919 *Fast = true;
18920 return true;
18921 }
18922
18923 // In little-endian MVE, the store instructions VSTRB.U8, VSTRH.U16 and
18924 // VSTRW.U32 all store the vector register in exactly the same format, and
18925 // differ only in the range of their immediate offset field and the required
18926 // alignment. So there is always a store that can be used, regardless of
18927 // actual type.
18928 //
18929 // For big endian, that is not the case. But can still emit a (VSTRB.U8;
18930 // VREV64.8) pair and get the same effect. This will likely be better than
18931 // aligning the vector through the stack.
18932 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
18933 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
18934 Ty == MVT::v2f64) {
18935 if (Fast)
18936 *Fast = 1;
18937 return true;
18938 }
18939
18940 return false;
18941 }
18942
18943
getOptimalMemOpType(const MemOp & Op,const AttributeList & FuncAttributes) const18944 EVT ARMTargetLowering::getOptimalMemOpType(
18945 const MemOp &Op, const AttributeList &FuncAttributes) const {
18946 // See if we can use NEON instructions for this...
18947 if ((Op.isMemcpy() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
18948 !FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
18949 unsigned Fast;
18950 if (Op.size() >= 16 &&
18951 (Op.isAligned(Align(16)) ||
18952 (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, Align(1),
18953 MachineMemOperand::MONone, &Fast) &&
18954 Fast))) {
18955 return MVT::v2f64;
18956 } else if (Op.size() >= 8 &&
18957 (Op.isAligned(Align(8)) ||
18958 (allowsMisalignedMemoryAccesses(
18959 MVT::f64, 0, Align(1), MachineMemOperand::MONone, &Fast) &&
18960 Fast))) {
18961 return MVT::f64;
18962 }
18963 }
18964
18965 // Let the target-independent logic figure it out.
18966 return MVT::Other;
18967 }
18968
18969 // 64-bit integers are split into their high and low parts and held in two
18970 // different registers, so the trunc is free since the low register can just
18971 // be used.
isTruncateFree(Type * SrcTy,Type * DstTy) const18972 bool ARMTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
18973 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
18974 return false;
18975 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
18976 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
18977 return (SrcBits == 64 && DestBits == 32);
18978 }
18979
isTruncateFree(EVT SrcVT,EVT DstVT) const18980 bool ARMTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
18981 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
18982 !DstVT.isInteger())
18983 return false;
18984 unsigned SrcBits = SrcVT.getSizeInBits();
18985 unsigned DestBits = DstVT.getSizeInBits();
18986 return (SrcBits == 64 && DestBits == 32);
18987 }
18988
isZExtFree(SDValue Val,EVT VT2) const18989 bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
18990 if (Val.getOpcode() != ISD::LOAD)
18991 return false;
18992
18993 EVT VT1 = Val.getValueType();
18994 if (!VT1.isSimple() || !VT1.isInteger() ||
18995 !VT2.isSimple() || !VT2.isInteger())
18996 return false;
18997
18998 switch (VT1.getSimpleVT().SimpleTy) {
18999 default: break;
19000 case MVT::i1:
19001 case MVT::i8:
19002 case MVT::i16:
19003 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
19004 return true;
19005 }
19006
19007 return false;
19008 }
19009
isFNegFree(EVT VT) const19010 bool ARMTargetLowering::isFNegFree(EVT VT) const {
19011 if (!VT.isSimple())
19012 return false;
19013
19014 // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that
19015 // negate values directly (fneg is free). So, we don't want to let the DAG
19016 // combiner rewrite fneg into xors and some other instructions. For f16 and
19017 // FullFP16 argument passing, some bitcast nodes may be introduced,
19018 // triggering this DAG combine rewrite, so we are avoiding that with this.
19019 switch (VT.getSimpleVT().SimpleTy) {
19020 default: break;
19021 case MVT::f16:
19022 return Subtarget->hasFullFP16();
19023 }
19024
19025 return false;
19026 }
19027
19028 /// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
19029 /// of the vector elements.
areExtractExts(Value * Ext1,Value * Ext2)19030 static bool areExtractExts(Value *Ext1, Value *Ext2) {
19031 auto areExtDoubled = [](Instruction *Ext) {
19032 return Ext->getType()->getScalarSizeInBits() ==
19033 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19034 };
19035
19036 if (!match(Ext1, m_ZExtOrSExt(m_Value())) ||
19037 !match(Ext2, m_ZExtOrSExt(m_Value())) ||
19038 !areExtDoubled(cast<Instruction>(Ext1)) ||
19039 !areExtDoubled(cast<Instruction>(Ext2)))
19040 return false;
19041
19042 return true;
19043 }
19044
19045 /// Check if sinking \p I's operands to I's basic block is profitable, because
19046 /// the operands can be folded into a target instruction, e.g.
19047 /// sext/zext can be folded into vsubl.
shouldSinkOperands(Instruction * I,SmallVectorImpl<Use * > & Ops) const19048 bool ARMTargetLowering::shouldSinkOperands(Instruction *I,
19049 SmallVectorImpl<Use *> &Ops) const {
19050 if (!I->getType()->isVectorTy())
19051 return false;
19052
19053 if (Subtarget->hasNEON()) {
19054 switch (I->getOpcode()) {
19055 case Instruction::Sub:
19056 case Instruction::Add: {
19057 if (!areExtractExts(I->getOperand(0), I->getOperand(1)))
19058 return false;
19059 Ops.push_back(&I->getOperandUse(0));
19060 Ops.push_back(&I->getOperandUse(1));
19061 return true;
19062 }
19063 default:
19064 return false;
19065 }
19066 }
19067
19068 if (!Subtarget->hasMVEIntegerOps())
19069 return false;
19070
19071 auto IsFMSMul = [&](Instruction *I) {
19072 if (!I->hasOneUse())
19073 return false;
19074 auto *Sub = cast<Instruction>(*I->users().begin());
19075 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) == I;
19076 };
19077 auto IsFMS = [&](Instruction *I) {
19078 if (match(I->getOperand(0), m_FNeg(m_Value())) ||
19079 match(I->getOperand(1), m_FNeg(m_Value())))
19080 return true;
19081 return false;
19082 };
19083
19084 auto IsSinker = [&](Instruction *I, int Operand) {
19085 switch (I->getOpcode()) {
19086 case Instruction::Add:
19087 case Instruction::Mul:
19088 case Instruction::FAdd:
19089 case Instruction::ICmp:
19090 case Instruction::FCmp:
19091 return true;
19092 case Instruction::FMul:
19093 return !IsFMSMul(I);
19094 case Instruction::Sub:
19095 case Instruction::FSub:
19096 case Instruction::Shl:
19097 case Instruction::LShr:
19098 case Instruction::AShr:
19099 return Operand == 1;
19100 case Instruction::Call:
19101 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
19102 switch (II->getIntrinsicID()) {
19103 case Intrinsic::fma:
19104 return !IsFMS(I);
19105 case Intrinsic::sadd_sat:
19106 case Intrinsic::uadd_sat:
19107 case Intrinsic::arm_mve_add_predicated:
19108 case Intrinsic::arm_mve_mul_predicated:
19109 case Intrinsic::arm_mve_qadd_predicated:
19110 case Intrinsic::arm_mve_vhadd:
19111 case Intrinsic::arm_mve_hadd_predicated:
19112 case Intrinsic::arm_mve_vqdmull:
19113 case Intrinsic::arm_mve_vqdmull_predicated:
19114 case Intrinsic::arm_mve_vqdmulh:
19115 case Intrinsic::arm_mve_qdmulh_predicated:
19116 case Intrinsic::arm_mve_vqrdmulh:
19117 case Intrinsic::arm_mve_qrdmulh_predicated:
19118 case Intrinsic::arm_mve_fma_predicated:
19119 return true;
19120 case Intrinsic::ssub_sat:
19121 case Intrinsic::usub_sat:
19122 case Intrinsic::arm_mve_sub_predicated:
19123 case Intrinsic::arm_mve_qsub_predicated:
19124 case Intrinsic::arm_mve_hsub_predicated:
19125 case Intrinsic::arm_mve_vhsub:
19126 return Operand == 1;
19127 default:
19128 return false;
19129 }
19130 }
19131 return false;
19132 default:
19133 return false;
19134 }
19135 };
19136
19137 for (auto OpIdx : enumerate(I->operands())) {
19138 Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
19139 // Make sure we are not already sinking this operand
19140 if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
19141 continue;
19142
19143 Instruction *Shuffle = Op;
19144 if (Shuffle->getOpcode() == Instruction::BitCast)
19145 Shuffle = dyn_cast<Instruction>(Shuffle->getOperand(0));
19146 // We are looking for a splat that can be sunk.
19147 if (!Shuffle ||
19148 !match(Shuffle, m_Shuffle(
19149 m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
19150 m_Undef(), m_ZeroMask())))
19151 continue;
19152 if (!IsSinker(I, OpIdx.index()))
19153 continue;
19154
19155 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
19156 // and vector registers
19157 for (Use &U : Op->uses()) {
19158 Instruction *Insn = cast<Instruction>(U.getUser());
19159 if (!IsSinker(Insn, U.getOperandNo()))
19160 return false;
19161 }
19162
19163 Ops.push_back(&Shuffle->getOperandUse(0));
19164 if (Shuffle != Op)
19165 Ops.push_back(&Op->getOperandUse(0));
19166 Ops.push_back(&OpIdx.value());
19167 }
19168 return true;
19169 }
19170
shouldConvertSplatType(ShuffleVectorInst * SVI) const19171 Type *ARMTargetLowering::shouldConvertSplatType(ShuffleVectorInst *SVI) const {
19172 if (!Subtarget->hasMVEIntegerOps())
19173 return nullptr;
19174 Type *SVIType = SVI->getType();
19175 Type *ScalarType = SVIType->getScalarType();
19176
19177 if (ScalarType->isFloatTy())
19178 return Type::getInt32Ty(SVIType->getContext());
19179 if (ScalarType->isHalfTy())
19180 return Type::getInt16Ty(SVIType->getContext());
19181 return nullptr;
19182 }
19183
isVectorLoadExtDesirable(SDValue ExtVal) const19184 bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
19185 EVT VT = ExtVal.getValueType();
19186
19187 if (!isTypeLegal(VT))
19188 return false;
19189
19190 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.getOperand(0))) {
19191 if (Ld->isExpandingLoad())
19192 return false;
19193 }
19194
19195 if (Subtarget->hasMVEIntegerOps())
19196 return true;
19197
19198 // Don't create a loadext if we can fold the extension into a wide/long
19199 // instruction.
19200 // If there's more than one user instruction, the loadext is desirable no
19201 // matter what. There can be two uses by the same instruction.
19202 if (ExtVal->use_empty() ||
19203 !ExtVal->use_begin()->isOnlyUserOf(ExtVal.getNode()))
19204 return true;
19205
19206 SDNode *U = *ExtVal->use_begin();
19207 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
19208 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM))
19209 return false;
19210
19211 return true;
19212 }
19213
allowTruncateForTailCall(Type * Ty1,Type * Ty2) const19214 bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
19215 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19216 return false;
19217
19218 if (!isTypeLegal(EVT::getEVT(Ty1)))
19219 return false;
19220
19221 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
19222
19223 // Assuming the caller doesn't have a zeroext or signext return parameter,
19224 // truncation all the way down to i1 is valid.
19225 return true;
19226 }
19227
19228 /// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
19229 /// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
19230 /// expanded to FMAs when this method returns true, otherwise fmuladd is
19231 /// expanded to fmul + fadd.
19232 ///
19233 /// ARM supports both fused and unfused multiply-add operations; we already
19234 /// lower a pair of fmul and fadd to the latter so it's not clear that there
19235 /// would be a gain or that the gain would be worthwhile enough to risk
19236 /// correctness bugs.
19237 ///
19238 /// For MVE, we set this to true as it helps simplify the need for some
19239 /// patterns (and we don't have the non-fused floating point instruction).
isFMAFasterThanFMulAndFAdd(const MachineFunction & MF,EVT VT) const19240 bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19241 EVT VT) const {
19242 if (!VT.isSimple())
19243 return false;
19244
19245 switch (VT.getSimpleVT().SimpleTy) {
19246 case MVT::v4f32:
19247 case MVT::v8f16:
19248 return Subtarget->hasMVEFloatOps();
19249 case MVT::f16:
19250 return Subtarget->useFPVFMx16();
19251 case MVT::f32:
19252 return Subtarget->useFPVFMx();
19253 case MVT::f64:
19254 return Subtarget->useFPVFMx64();
19255 default:
19256 break;
19257 }
19258
19259 return false;
19260 }
19261
isLegalT1AddressImmediate(int64_t V,EVT VT)19262 static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
19263 if (V < 0)
19264 return false;
19265
19266 unsigned Scale = 1;
19267 switch (VT.getSimpleVT().SimpleTy) {
19268 case MVT::i1:
19269 case MVT::i8:
19270 // Scale == 1;
19271 break;
19272 case MVT::i16:
19273 // Scale == 2;
19274 Scale = 2;
19275 break;
19276 default:
19277 // On thumb1 we load most things (i32, i64, floats, etc) with a LDR
19278 // Scale == 4;
19279 Scale = 4;
19280 break;
19281 }
19282
19283 if ((V & (Scale - 1)) != 0)
19284 return false;
19285 return isUInt<5>(V / Scale);
19286 }
19287
isLegalT2AddressImmediate(int64_t V,EVT VT,const ARMSubtarget * Subtarget)19288 static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
19289 const ARMSubtarget *Subtarget) {
19290 if (!VT.isInteger() && !VT.isFloatingPoint())
19291 return false;
19292 if (VT.isVector() && Subtarget->hasNEON())
19293 return false;
19294 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() &&
19295 !Subtarget->hasMVEFloatOps())
19296 return false;
19297
19298 bool IsNeg = false;
19299 if (V < 0) {
19300 IsNeg = true;
19301 V = -V;
19302 }
19303
19304 unsigned NumBytes = std::max((unsigned)VT.getSizeInBits() / 8, 1U);
19305
19306 // MVE: size * imm7
19307 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) {
19308 switch (VT.getSimpleVT().getVectorElementType().SimpleTy) {
19309 case MVT::i32:
19310 case MVT::f32:
19311 return isShiftedUInt<7,2>(V);
19312 case MVT::i16:
19313 case MVT::f16:
19314 return isShiftedUInt<7,1>(V);
19315 case MVT::i8:
19316 return isUInt<7>(V);
19317 default:
19318 return false;
19319 }
19320 }
19321
19322 // half VLDR: 2 * imm8
19323 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19324 return isShiftedUInt<8, 1>(V);
19325 // VLDR and LDRD: 4 * imm8
19326 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8)
19327 return isShiftedUInt<8, 2>(V);
19328
19329 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19330 // + imm12 or - imm8
19331 if (IsNeg)
19332 return isUInt<8>(V);
19333 return isUInt<12>(V);
19334 }
19335
19336 return false;
19337 }
19338
19339 /// isLegalAddressImmediate - Return true if the integer value can be used
19340 /// as the offset of the target addressing mode for load / store of the
19341 /// given type.
isLegalAddressImmediate(int64_t V,EVT VT,const ARMSubtarget * Subtarget)19342 static bool isLegalAddressImmediate(int64_t V, EVT VT,
19343 const ARMSubtarget *Subtarget) {
19344 if (V == 0)
19345 return true;
19346
19347 if (!VT.isSimple())
19348 return false;
19349
19350 if (Subtarget->isThumb1Only())
19351 return isLegalT1AddressImmediate(V, VT);
19352 else if (Subtarget->isThumb2())
19353 return isLegalT2AddressImmediate(V, VT, Subtarget);
19354
19355 // ARM mode.
19356 if (V < 0)
19357 V = - V;
19358 switch (VT.getSimpleVT().SimpleTy) {
19359 default: return false;
19360 case MVT::i1:
19361 case MVT::i8:
19362 case MVT::i32:
19363 // +- imm12
19364 return isUInt<12>(V);
19365 case MVT::i16:
19366 // +- imm8
19367 return isUInt<8>(V);
19368 case MVT::f32:
19369 case MVT::f64:
19370 if (!Subtarget->hasVFP2Base()) // FIXME: NEON?
19371 return false;
19372 return isShiftedUInt<8, 2>(V);
19373 }
19374 }
19375
isLegalT2ScaledAddressingMode(const AddrMode & AM,EVT VT) const19376 bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
19377 EVT VT) const {
19378 int Scale = AM.Scale;
19379 if (Scale < 0)
19380 return false;
19381
19382 switch (VT.getSimpleVT().SimpleTy) {
19383 default: return false;
19384 case MVT::i1:
19385 case MVT::i8:
19386 case MVT::i16:
19387 case MVT::i32:
19388 if (Scale == 1)
19389 return true;
19390 // r + r << imm
19391 Scale = Scale & ~1;
19392 return Scale == 2 || Scale == 4 || Scale == 8;
19393 case MVT::i64:
19394 // FIXME: What are we trying to model here? ldrd doesn't have an r + r
19395 // version in Thumb mode.
19396 // r + r
19397 if (Scale == 1)
19398 return true;
19399 // r * 2 (this can be lowered to r + r).
19400 if (!AM.HasBaseReg && Scale == 2)
19401 return true;
19402 return false;
19403 case MVT::isVoid:
19404 // Note, we allow "void" uses (basically, uses that aren't loads or
19405 // stores), because arm allows folding a scale into many arithmetic
19406 // operations. This should be made more precise and revisited later.
19407
19408 // Allow r << imm, but the imm has to be a multiple of two.
19409 if (Scale & 1) return false;
19410 return isPowerOf2_32(Scale);
19411 }
19412 }
19413
isLegalT1ScaledAddressingMode(const AddrMode & AM,EVT VT) const19414 bool ARMTargetLowering::isLegalT1ScaledAddressingMode(const AddrMode &AM,
19415 EVT VT) const {
19416 const int Scale = AM.Scale;
19417
19418 // Negative scales are not supported in Thumb1.
19419 if (Scale < 0)
19420 return false;
19421
19422 // Thumb1 addressing modes do not support register scaling excepting the
19423 // following cases:
19424 // 1. Scale == 1 means no scaling.
19425 // 2. Scale == 2 this can be lowered to r + r if there is no base register.
19426 return (Scale == 1) || (!AM.HasBaseReg && Scale == 2);
19427 }
19428
19429 /// isLegalAddressingMode - Return true if the addressing mode represented
19430 /// by AM is legal for this target, for a load/store of the specified type.
isLegalAddressingMode(const DataLayout & DL,const AddrMode & AM,Type * Ty,unsigned AS,Instruction * I) const19431 bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
19432 const AddrMode &AM, Type *Ty,
19433 unsigned AS, Instruction *I) const {
19434 EVT VT = getValueType(DL, Ty, true);
19435 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
19436 return false;
19437
19438 // Can never fold addr of global into load/store.
19439 if (AM.BaseGV)
19440 return false;
19441
19442 switch (AM.Scale) {
19443 case 0: // no scale reg, must be "r+i" or "r", or "i".
19444 break;
19445 default:
19446 // ARM doesn't support any R+R*scale+imm addr modes.
19447 if (AM.BaseOffs)
19448 return false;
19449
19450 if (!VT.isSimple())
19451 return false;
19452
19453 if (Subtarget->isThumb1Only())
19454 return isLegalT1ScaledAddressingMode(AM, VT);
19455
19456 if (Subtarget->isThumb2())
19457 return isLegalT2ScaledAddressingMode(AM, VT);
19458
19459 int Scale = AM.Scale;
19460 switch (VT.getSimpleVT().SimpleTy) {
19461 default: return false;
19462 case MVT::i1:
19463 case MVT::i8:
19464 case MVT::i32:
19465 if (Scale < 0) Scale = -Scale;
19466 if (Scale == 1)
19467 return true;
19468 // r + r << imm
19469 return isPowerOf2_32(Scale & ~1);
19470 case MVT::i16:
19471 case MVT::i64:
19472 // r +/- r
19473 if (Scale == 1 || (AM.HasBaseReg && Scale == -1))
19474 return true;
19475 // r * 2 (this can be lowered to r + r).
19476 if (!AM.HasBaseReg && Scale == 2)
19477 return true;
19478 return false;
19479
19480 case MVT::isVoid:
19481 // Note, we allow "void" uses (basically, uses that aren't loads or
19482 // stores), because arm allows folding a scale into many arithmetic
19483 // operations. This should be made more precise and revisited later.
19484
19485 // Allow r << imm, but the imm has to be a multiple of two.
19486 if (Scale & 1) return false;
19487 return isPowerOf2_32(Scale);
19488 }
19489 }
19490 return true;
19491 }
19492
19493 /// isLegalICmpImmediate - Return true if the specified immediate is legal
19494 /// icmp immediate, that is the target has icmp instructions which can compare
19495 /// a register against the immediate without having to materialize the
19496 /// immediate into a register.
isLegalICmpImmediate(int64_t Imm) const19497 bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
19498 // Thumb2 and ARM modes can use cmn for negative immediates.
19499 if (!Subtarget->isThumb())
19500 return ARM_AM::getSOImmVal((uint32_t)Imm) != -1 ||
19501 ARM_AM::getSOImmVal(-(uint32_t)Imm) != -1;
19502 if (Subtarget->isThumb2())
19503 return ARM_AM::getT2SOImmVal((uint32_t)Imm) != -1 ||
19504 ARM_AM::getT2SOImmVal(-(uint32_t)Imm) != -1;
19505 // Thumb1 doesn't have cmn, and only 8-bit immediates.
19506 return Imm >= 0 && Imm <= 255;
19507 }
19508
19509 /// isLegalAddImmediate - Return true if the specified immediate is a legal add
19510 /// *or sub* immediate, that is the target has add or sub instructions which can
19511 /// add a register with the immediate without having to materialize the
19512 /// immediate into a register.
isLegalAddImmediate(int64_t Imm) const19513 bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const {
19514 // Same encoding for add/sub, just flip the sign.
19515 int64_t AbsImm = std::abs(Imm);
19516 if (!Subtarget->isThumb())
19517 return ARM_AM::getSOImmVal(AbsImm) != -1;
19518 if (Subtarget->isThumb2())
19519 return ARM_AM::getT2SOImmVal(AbsImm) != -1;
19520 // Thumb1 only has 8-bit unsigned immediate.
19521 return AbsImm >= 0 && AbsImm <= 255;
19522 }
19523
19524 // Return false to prevent folding
19525 // (mul (add r, c0), c1) -> (add (mul r, c1), c0*c1) in DAGCombine,
19526 // if the folding leads to worse code.
isMulAddWithConstProfitable(SDValue AddNode,SDValue ConstNode) const19527 bool ARMTargetLowering::isMulAddWithConstProfitable(SDValue AddNode,
19528 SDValue ConstNode) const {
19529 // Let the DAGCombiner decide for vector types and large types.
19530 const EVT VT = AddNode.getValueType();
19531 if (VT.isVector() || VT.getScalarSizeInBits() > 32)
19532 return true;
19533
19534 // It is worse if c0 is legal add immediate, while c1*c0 is not
19535 // and has to be composed by at least two instructions.
19536 const ConstantSDNode *C0Node = cast<ConstantSDNode>(AddNode.getOperand(1));
19537 const ConstantSDNode *C1Node = cast<ConstantSDNode>(ConstNode);
19538 const int64_t C0 = C0Node->getSExtValue();
19539 APInt CA = C0Node->getAPIntValue() * C1Node->getAPIntValue();
19540 if (!isLegalAddImmediate(C0) || isLegalAddImmediate(CA.getSExtValue()))
19541 return true;
19542 if (ConstantMaterializationCost((unsigned)CA.getZExtValue(), Subtarget) > 1)
19543 return false;
19544
19545 // Default to true and let the DAGCombiner decide.
19546 return true;
19547 }
19548
getARMIndexedAddressParts(SDNode * Ptr,EVT VT,bool isSEXTLoad,SDValue & Base,SDValue & Offset,bool & isInc,SelectionDAG & DAG)19549 static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
19550 bool isSEXTLoad, SDValue &Base,
19551 SDValue &Offset, bool &isInc,
19552 SelectionDAG &DAG) {
19553 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19554 return false;
19555
19556 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19557 // AddressingMode 3
19558 Base = Ptr->getOperand(0);
19559 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
19560 int RHSC = (int)RHS->getZExtValue();
19561 if (RHSC < 0 && RHSC > -256) {
19562 assert(Ptr->getOpcode() == ISD::ADD);
19563 isInc = false;
19564 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19565 return true;
19566 }
19567 }
19568 isInc = (Ptr->getOpcode() == ISD::ADD);
19569 Offset = Ptr->getOperand(1);
19570 return true;
19571 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19572 // AddressingMode 2
19573 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
19574 int RHSC = (int)RHS->getZExtValue();
19575 if (RHSC < 0 && RHSC > -0x1000) {
19576 assert(Ptr->getOpcode() == ISD::ADD);
19577 isInc = false;
19578 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19579 Base = Ptr->getOperand(0);
19580 return true;
19581 }
19582 }
19583
19584 if (Ptr->getOpcode() == ISD::ADD) {
19585 isInc = true;
19586 ARM_AM::ShiftOpc ShOpcVal=
19587 ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode());
19588 if (ShOpcVal != ARM_AM::no_shift) {
19589 Base = Ptr->getOperand(1);
19590 Offset = Ptr->getOperand(0);
19591 } else {
19592 Base = Ptr->getOperand(0);
19593 Offset = Ptr->getOperand(1);
19594 }
19595 return true;
19596 }
19597
19598 isInc = (Ptr->getOpcode() == ISD::ADD);
19599 Base = Ptr->getOperand(0);
19600 Offset = Ptr->getOperand(1);
19601 return true;
19602 }
19603
19604 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
19605 return false;
19606 }
19607
getT2IndexedAddressParts(SDNode * Ptr,EVT VT,bool isSEXTLoad,SDValue & Base,SDValue & Offset,bool & isInc,SelectionDAG & DAG)19608 static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
19609 bool isSEXTLoad, SDValue &Base,
19610 SDValue &Offset, bool &isInc,
19611 SelectionDAG &DAG) {
19612 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19613 return false;
19614
19615 Base = Ptr->getOperand(0);
19616 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
19617 int RHSC = (int)RHS->getZExtValue();
19618 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
19619 assert(Ptr->getOpcode() == ISD::ADD);
19620 isInc = false;
19621 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19622 return true;
19623 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
19624 isInc = Ptr->getOpcode() == ISD::ADD;
19625 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
19626 return true;
19627 }
19628 }
19629
19630 return false;
19631 }
19632
getMVEIndexedAddressParts(SDNode * Ptr,EVT VT,Align Alignment,bool isSEXTLoad,bool IsMasked,bool isLE,SDValue & Base,SDValue & Offset,bool & isInc,SelectionDAG & DAG)19633 static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment,
19634 bool isSEXTLoad, bool IsMasked, bool isLE,
19635 SDValue &Base, SDValue &Offset,
19636 bool &isInc, SelectionDAG &DAG) {
19637 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19638 return false;
19639 if (!isa<ConstantSDNode>(Ptr->getOperand(1)))
19640 return false;
19641
19642 // We allow LE non-masked loads to change the type (for example use a vldrb.8
19643 // as opposed to a vldrw.32). This can allow extra addressing modes or
19644 // alignments for what is otherwise an equivalent instruction.
19645 bool CanChangeType = isLE && !IsMasked;
19646
19647 ConstantSDNode *RHS = cast<ConstantSDNode>(Ptr->getOperand(1));
19648 int RHSC = (int)RHS->getZExtValue();
19649
19650 auto IsInRange = [&](int RHSC, int Limit, int Scale) {
19651 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19652 assert(Ptr->getOpcode() == ISD::ADD);
19653 isInc = false;
19654 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19655 return true;
19656 } else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19657 isInc = Ptr->getOpcode() == ISD::ADD;
19658 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
19659 return true;
19660 }
19661 return false;
19662 };
19663
19664 // Try to find a matching instruction based on s/zext, Alignment, Offset and
19665 // (in BE/masked) type.
19666 Base = Ptr->getOperand(0);
19667 if (VT == MVT::v4i16) {
19668 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19669 return true;
19670 } else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19671 if (IsInRange(RHSC, 0x80, 1))
19672 return true;
19673 } else if (Alignment >= 4 &&
19674 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19675 IsInRange(RHSC, 0x80, 4))
19676 return true;
19677 else if (Alignment >= 2 &&
19678 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19679 IsInRange(RHSC, 0x80, 2))
19680 return true;
19681 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19682 return true;
19683 return false;
19684 }
19685
19686 /// getPreIndexedAddressParts - returns true by value, base pointer and
19687 /// offset pointer and addressing mode by reference if the node's address
19688 /// can be legally represented as pre-indexed load / store address.
19689 bool
getPreIndexedAddressParts(SDNode * N,SDValue & Base,SDValue & Offset,ISD::MemIndexedMode & AM,SelectionDAG & DAG) const19690 ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
19691 SDValue &Offset,
19692 ISD::MemIndexedMode &AM,
19693 SelectionDAG &DAG) const {
19694 if (Subtarget->isThumb1Only())
19695 return false;
19696
19697 EVT VT;
19698 SDValue Ptr;
19699 Align Alignment;
19700 bool isSEXTLoad = false;
19701 bool IsMasked = false;
19702 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
19703 Ptr = LD->getBasePtr();
19704 VT = LD->getMemoryVT();
19705 Alignment = LD->getAlign();
19706 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19707 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
19708 Ptr = ST->getBasePtr();
19709 VT = ST->getMemoryVT();
19710 Alignment = ST->getAlign();
19711 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
19712 Ptr = LD->getBasePtr();
19713 VT = LD->getMemoryVT();
19714 Alignment = LD->getAlign();
19715 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19716 IsMasked = true;
19717 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(N)) {
19718 Ptr = ST->getBasePtr();
19719 VT = ST->getMemoryVT();
19720 Alignment = ST->getAlign();
19721 IsMasked = true;
19722 } else
19723 return false;
19724
19725 bool isInc;
19726 bool isLegal = false;
19727 if (VT.isVector())
19728 isLegal = Subtarget->hasMVEIntegerOps() &&
19729 getMVEIndexedAddressParts(
19730 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
19731 Subtarget->isLittle(), Base, Offset, isInc, DAG);
19732 else {
19733 if (Subtarget->isThumb2())
19734 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
19735 Offset, isInc, DAG);
19736 else
19737 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
19738 Offset, isInc, DAG);
19739 }
19740 if (!isLegal)
19741 return false;
19742
19743 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
19744 return true;
19745 }
19746
19747 /// getPostIndexedAddressParts - returns true by value, base pointer and
19748 /// offset pointer and addressing mode by reference if this node can be
19749 /// combined with a load / store to form a post-indexed load / store.
getPostIndexedAddressParts(SDNode * N,SDNode * Op,SDValue & Base,SDValue & Offset,ISD::MemIndexedMode & AM,SelectionDAG & DAG) const19750 bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
19751 SDValue &Base,
19752 SDValue &Offset,
19753 ISD::MemIndexedMode &AM,
19754 SelectionDAG &DAG) const {
19755 EVT VT;
19756 SDValue Ptr;
19757 Align Alignment;
19758 bool isSEXTLoad = false, isNonExt;
19759 bool IsMasked = false;
19760 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
19761 VT = LD->getMemoryVT();
19762 Ptr = LD->getBasePtr();
19763 Alignment = LD->getAlign();
19764 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19765 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
19766 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
19767 VT = ST->getMemoryVT();
19768 Ptr = ST->getBasePtr();
19769 Alignment = ST->getAlign();
19770 isNonExt = !ST->isTruncatingStore();
19771 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
19772 VT = LD->getMemoryVT();
19773 Ptr = LD->getBasePtr();
19774 Alignment = LD->getAlign();
19775 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19776 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
19777 IsMasked = true;
19778 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(N)) {
19779 VT = ST->getMemoryVT();
19780 Ptr = ST->getBasePtr();
19781 Alignment = ST->getAlign();
19782 isNonExt = !ST->isTruncatingStore();
19783 IsMasked = true;
19784 } else
19785 return false;
19786
19787 if (Subtarget->isThumb1Only()) {
19788 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It
19789 // must be non-extending/truncating, i32, with an offset of 4.
19790 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!");
19791 if (Op->getOpcode() != ISD::ADD || !isNonExt)
19792 return false;
19793 auto *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1));
19794 if (!RHS || RHS->getZExtValue() != 4)
19795 return false;
19796 if (Alignment < Align(4))
19797 return false;
19798
19799 Offset = Op->getOperand(1);
19800 Base = Op->getOperand(0);
19801 AM = ISD::POST_INC;
19802 return true;
19803 }
19804
19805 bool isInc;
19806 bool isLegal = false;
19807 if (VT.isVector())
19808 isLegal = Subtarget->hasMVEIntegerOps() &&
19809 getMVEIndexedAddressParts(Op, VT, Alignment, isSEXTLoad, IsMasked,
19810 Subtarget->isLittle(), Base, Offset,
19811 isInc, DAG);
19812 else {
19813 if (Subtarget->isThumb2())
19814 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
19815 isInc, DAG);
19816 else
19817 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
19818 isInc, DAG);
19819 }
19820 if (!isLegal)
19821 return false;
19822
19823 if (Ptr != Base) {
19824 // Swap base ptr and offset to catch more post-index load / store when
19825 // it's legal. In Thumb2 mode, offset must be an immediate.
19826 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
19827 !Subtarget->isThumb2())
19828 std::swap(Base, Offset);
19829
19830 // Post-indexed load / store update the base pointer.
19831 if (Ptr != Base)
19832 return false;
19833 }
19834
19835 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
19836 return true;
19837 }
19838
computeKnownBitsForTargetNode(const SDValue Op,KnownBits & Known,const APInt & DemandedElts,const SelectionDAG & DAG,unsigned Depth) const19839 void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
19840 KnownBits &Known,
19841 const APInt &DemandedElts,
19842 const SelectionDAG &DAG,
19843 unsigned Depth) const {
19844 unsigned BitWidth = Known.getBitWidth();
19845 Known.resetAll();
19846 switch (Op.getOpcode()) {
19847 default: break;
19848 case ARMISD::ADDC:
19849 case ARMISD::ADDE:
19850 case ARMISD::SUBC:
19851 case ARMISD::SUBE:
19852 // Special cases when we convert a carry to a boolean.
19853 if (Op.getResNo() == 0) {
19854 SDValue LHS = Op.getOperand(0);
19855 SDValue RHS = Op.getOperand(1);
19856 // (ADDE 0, 0, C) will give us a single bit.
19857 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(LHS) &&
19858 isNullConstant(RHS)) {
19859 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
19860 return;
19861 }
19862 }
19863 break;
19864 case ARMISD::CMOV: {
19865 // Bits are known zero/one if known on the LHS and RHS.
19866 Known = DAG.computeKnownBits(Op.getOperand(0), Depth+1);
19867 if (Known.isUnknown())
19868 return;
19869
19870 KnownBits KnownRHS = DAG.computeKnownBits(Op.getOperand(1), Depth+1);
19871 Known = KnownBits::commonBits(Known, KnownRHS);
19872 return;
19873 }
19874 case ISD::INTRINSIC_W_CHAIN: {
19875 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
19876 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
19877 switch (IntID) {
19878 default: return;
19879 case Intrinsic::arm_ldaex:
19880 case Intrinsic::arm_ldrex: {
19881 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
19882 unsigned MemBits = VT.getScalarSizeInBits();
19883 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
19884 return;
19885 }
19886 }
19887 }
19888 case ARMISD::BFI: {
19889 // Conservatively, we can recurse down the first operand
19890 // and just mask out all affected bits.
19891 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
19892
19893 // The operand to BFI is already a mask suitable for removing the bits it
19894 // sets.
19895 ConstantSDNode *CI = cast<ConstantSDNode>(Op.getOperand(2));
19896 const APInt &Mask = CI->getAPIntValue();
19897 Known.Zero &= Mask;
19898 Known.One &= Mask;
19899 return;
19900 }
19901 case ARMISD::VGETLANEs:
19902 case ARMISD::VGETLANEu: {
19903 const SDValue &SrcSV = Op.getOperand(0);
19904 EVT VecVT = SrcSV.getValueType();
19905 assert(VecVT.isVector() && "VGETLANE expected a vector type");
19906 const unsigned NumSrcElts = VecVT.getVectorNumElements();
19907 ConstantSDNode *Pos = cast<ConstantSDNode>(Op.getOperand(1).getNode());
19908 assert(Pos->getAPIntValue().ult(NumSrcElts) &&
19909 "VGETLANE index out of bounds");
19910 unsigned Idx = Pos->getZExtValue();
19911 APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
19912 Known = DAG.computeKnownBits(SrcSV, DemandedElt, Depth + 1);
19913
19914 EVT VT = Op.getValueType();
19915 const unsigned DstSz = VT.getScalarSizeInBits();
19916 const unsigned SrcSz = VecVT.getVectorElementType().getSizeInBits();
19917 (void)SrcSz;
19918 assert(SrcSz == Known.getBitWidth());
19919 assert(DstSz > SrcSz);
19920 if (Op.getOpcode() == ARMISD::VGETLANEs)
19921 Known = Known.sext(DstSz);
19922 else {
19923 Known = Known.zext(DstSz);
19924 }
19925 assert(DstSz == Known.getBitWidth());
19926 break;
19927 }
19928 case ARMISD::VMOVrh: {
19929 KnownBits KnownOp = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
19930 assert(KnownOp.getBitWidth() == 16);
19931 Known = KnownOp.zext(32);
19932 break;
19933 }
19934 case ARMISD::CSINC:
19935 case ARMISD::CSINV:
19936 case ARMISD::CSNEG: {
19937 KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
19938 KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
19939
19940 // The result is either:
19941 // CSINC: KnownOp0 or KnownOp1 + 1
19942 // CSINV: KnownOp0 or ~KnownOp1
19943 // CSNEG: KnownOp0 or KnownOp1 * -1
19944 if (Op.getOpcode() == ARMISD::CSINC)
19945 KnownOp1 = KnownBits::computeForAddSub(
19946 true, false, KnownOp1, KnownBits::makeConstant(APInt(32, 1)));
19947 else if (Op.getOpcode() == ARMISD::CSINV)
19948 std::swap(KnownOp1.Zero, KnownOp1.One);
19949 else if (Op.getOpcode() == ARMISD::CSNEG)
19950 KnownOp1 = KnownBits::mul(
19951 KnownOp1, KnownBits::makeConstant(APInt(32, -1)));
19952
19953 Known = KnownBits::commonBits(KnownOp0, KnownOp1);
19954 break;
19955 }
19956 }
19957 }
19958
targetShrinkDemandedConstant(SDValue Op,const APInt & DemandedBits,const APInt & DemandedElts,TargetLoweringOpt & TLO) const19959 bool ARMTargetLowering::targetShrinkDemandedConstant(
19960 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
19961 TargetLoweringOpt &TLO) const {
19962 // Delay optimization, so we don't have to deal with illegal types, or block
19963 // optimizations.
19964 if (!TLO.LegalOps)
19965 return false;
19966
19967 // Only optimize AND for now.
19968 if (Op.getOpcode() != ISD::AND)
19969 return false;
19970
19971 EVT VT = Op.getValueType();
19972
19973 // Ignore vectors.
19974 if (VT.isVector())
19975 return false;
19976
19977 assert(VT == MVT::i32 && "Unexpected integer type");
19978
19979 // Make sure the RHS really is a constant.
19980 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
19981 if (!C)
19982 return false;
19983
19984 unsigned Mask = C->getZExtValue();
19985
19986 unsigned Demanded = DemandedBits.getZExtValue();
19987 unsigned ShrunkMask = Mask & Demanded;
19988 unsigned ExpandedMask = Mask | ~Demanded;
19989
19990 // If the mask is all zeros, let the target-independent code replace the
19991 // result with zero.
19992 if (ShrunkMask == 0)
19993 return false;
19994
19995 // If the mask is all ones, erase the AND. (Currently, the target-independent
19996 // code won't do this, so we have to do it explicitly to avoid an infinite
19997 // loop in obscure cases.)
19998 if (ExpandedMask == ~0U)
19999 return TLO.CombineTo(Op, Op.getOperand(0));
20000
20001 auto IsLegalMask = [ShrunkMask, ExpandedMask](unsigned Mask) -> bool {
20002 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20003 };
20004 auto UseMask = [Mask, Op, VT, &TLO](unsigned NewMask) -> bool {
20005 if (NewMask == Mask)
20006 return true;
20007 SDLoc DL(Op);
20008 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
20009 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
20010 return TLO.CombineTo(Op, NewOp);
20011 };
20012
20013 // Prefer uxtb mask.
20014 if (IsLegalMask(0xFF))
20015 return UseMask(0xFF);
20016
20017 // Prefer uxth mask.
20018 if (IsLegalMask(0xFFFF))
20019 return UseMask(0xFFFF);
20020
20021 // [1, 255] is Thumb1 movs+ands, legal immediate for ARM/Thumb2.
20022 // FIXME: Prefer a contiguous sequence of bits for other optimizations.
20023 if (ShrunkMask < 256)
20024 return UseMask(ShrunkMask);
20025
20026 // [-256, -2] is Thumb1 movs+bics, legal immediate for ARM/Thumb2.
20027 // FIXME: Prefer a contiguous sequence of bits for other optimizations.
20028 if ((int)ExpandedMask <= -2 && (int)ExpandedMask >= -256)
20029 return UseMask(ExpandedMask);
20030
20031 // Potential improvements:
20032 //
20033 // We could try to recognize lsls+lsrs or lsrs+lsls pairs here.
20034 // We could try to prefer Thumb1 immediates which can be lowered to a
20035 // two-instruction sequence.
20036 // We could try to recognize more legal ARM/Thumb2 immediates here.
20037
20038 return false;
20039 }
20040
SimplifyDemandedBitsForTargetNode(SDValue Op,const APInt & OriginalDemandedBits,const APInt & OriginalDemandedElts,KnownBits & Known,TargetLoweringOpt & TLO,unsigned Depth) const20041 bool ARMTargetLowering::SimplifyDemandedBitsForTargetNode(
20042 SDValue Op, const APInt &OriginalDemandedBits,
20043 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
20044 unsigned Depth) const {
20045 unsigned Opc = Op.getOpcode();
20046
20047 switch (Opc) {
20048 case ARMISD::ASRL:
20049 case ARMISD::LSRL: {
20050 // If this is result 0 and the other result is unused, see if the demand
20051 // bits allow us to shrink this long shift into a standard small shift in
20052 // the opposite direction.
20053 if (Op.getResNo() == 0 && !Op->hasAnyUseOfValue(1) &&
20054 isa<ConstantSDNode>(Op->getOperand(2))) {
20055 unsigned ShAmt = Op->getConstantOperandVal(2);
20056 if (ShAmt < 32 && OriginalDemandedBits.isSubsetOf(APInt::getAllOnes(32)
20057 << (32 - ShAmt)))
20058 return TLO.CombineTo(
20059 Op, TLO.DAG.getNode(
20060 ISD::SHL, SDLoc(Op), MVT::i32, Op.getOperand(1),
20061 TLO.DAG.getConstant(32 - ShAmt, SDLoc(Op), MVT::i32)));
20062 }
20063 break;
20064 }
20065 case ARMISD::VBICIMM: {
20066 SDValue Op0 = Op.getOperand(0);
20067 unsigned ModImm = Op.getConstantOperandVal(1);
20068 unsigned EltBits = 0;
20069 uint64_t Mask = ARM_AM::decodeVMOVModImm(ModImm, EltBits);
20070 if ((OriginalDemandedBits & Mask) == 0)
20071 return TLO.CombineTo(Op, Op0);
20072 }
20073 }
20074
20075 return TargetLowering::SimplifyDemandedBitsForTargetNode(
20076 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
20077 }
20078
20079 //===----------------------------------------------------------------------===//
20080 // ARM Inline Assembly Support
20081 //===----------------------------------------------------------------------===//
20082
ExpandInlineAsm(CallInst * CI) const20083 bool ARMTargetLowering::ExpandInlineAsm(CallInst *CI) const {
20084 // Looking for "rev" which is V6+.
20085 if (!Subtarget->hasV6Ops())
20086 return false;
20087
20088 InlineAsm *IA = cast<InlineAsm>(CI->getCalledOperand());
20089 std::string AsmStr = IA->getAsmString();
20090 SmallVector<StringRef, 4> AsmPieces;
20091 SplitString(AsmStr, AsmPieces, ";\n");
20092
20093 switch (AsmPieces.size()) {
20094 default: return false;
20095 case 1:
20096 AsmStr = std::string(AsmPieces[0]);
20097 AsmPieces.clear();
20098 SplitString(AsmStr, AsmPieces, " \t,");
20099
20100 // rev $0, $1
20101 if (AsmPieces.size() == 3 &&
20102 AsmPieces[0] == "rev" && AsmPieces[1] == "$0" && AsmPieces[2] == "$1" &&
20103 IA->getConstraintString().compare(0, 4, "=l,l") == 0) {
20104 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
20105 if (Ty && Ty->getBitWidth() == 32)
20106 return IntrinsicLowering::LowerToByteSwap(CI);
20107 }
20108 break;
20109 }
20110
20111 return false;
20112 }
20113
LowerXConstraint(EVT ConstraintVT) const20114 const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const {
20115 // At this point, we have to lower this constraint to something else, so we
20116 // lower it to an "r" or "w". However, by doing this we will force the result
20117 // to be in register, while the X constraint is much more permissive.
20118 //
20119 // Although we are correct (we are free to emit anything, without
20120 // constraints), we might break use cases that would expect us to be more
20121 // efficient and emit something else.
20122 if (!Subtarget->hasVFP2Base())
20123 return "r";
20124 if (ConstraintVT.isFloatingPoint())
20125 return "w";
20126 if (ConstraintVT.isVector() && Subtarget->hasNEON() &&
20127 (ConstraintVT.getSizeInBits() == 64 ||
20128 ConstraintVT.getSizeInBits() == 128))
20129 return "w";
20130
20131 return "r";
20132 }
20133
20134 /// getConstraintType - Given a constraint letter, return the type of
20135 /// constraint it is for this target.
20136 ARMTargetLowering::ConstraintType
getConstraintType(StringRef Constraint) const20137 ARMTargetLowering::getConstraintType(StringRef Constraint) const {
20138 unsigned S = Constraint.size();
20139 if (S == 1) {
20140 switch (Constraint[0]) {
20141 default: break;
20142 case 'l': return C_RegisterClass;
20143 case 'w': return C_RegisterClass;
20144 case 'h': return C_RegisterClass;
20145 case 'x': return C_RegisterClass;
20146 case 't': return C_RegisterClass;
20147 case 'j': return C_Immediate; // Constant for movw.
20148 // An address with a single base register. Due to the way we
20149 // currently handle addresses it is the same as an 'r' memory constraint.
20150 case 'Q': return C_Memory;
20151 }
20152 } else if (S == 2) {
20153 switch (Constraint[0]) {
20154 default: break;
20155 case 'T': return C_RegisterClass;
20156 // All 'U+' constraints are addresses.
20157 case 'U': return C_Memory;
20158 }
20159 }
20160 return TargetLowering::getConstraintType(Constraint);
20161 }
20162
20163 /// Examine constraint type and operand type and determine a weight value.
20164 /// This object must already have been set up with the operand type
20165 /// and the current alternative constraint selected.
20166 TargetLowering::ConstraintWeight
getSingleConstraintMatchWeight(AsmOperandInfo & info,const char * constraint) const20167 ARMTargetLowering::getSingleConstraintMatchWeight(
20168 AsmOperandInfo &info, const char *constraint) const {
20169 ConstraintWeight weight = CW_Invalid;
20170 Value *CallOperandVal = info.CallOperandVal;
20171 // If we don't have a value, we can't do a match,
20172 // but allow it at the lowest weight.
20173 if (!CallOperandVal)
20174 return CW_Default;
20175 Type *type = CallOperandVal->getType();
20176 // Look at the constraint type.
20177 switch (*constraint) {
20178 default:
20179 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
20180 break;
20181 case 'l':
20182 if (type->isIntegerTy()) {
20183 if (Subtarget->isThumb())
20184 weight = CW_SpecificReg;
20185 else
20186 weight = CW_Register;
20187 }
20188 break;
20189 case 'w':
20190 if (type->isFloatingPointTy())
20191 weight = CW_Register;
20192 break;
20193 }
20194 return weight;
20195 }
20196
20197 using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20198
getRegForInlineAsmConstraint(const TargetRegisterInfo * TRI,StringRef Constraint,MVT VT) const20199 RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
20200 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
20201 switch (Constraint.size()) {
20202 case 1:
20203 // GCC ARM Constraint Letters
20204 switch (Constraint[0]) {
20205 case 'l': // Low regs or general regs.
20206 if (Subtarget->isThumb())
20207 return RCPair(0U, &ARM::tGPRRegClass);
20208 return RCPair(0U, &ARM::GPRRegClass);
20209 case 'h': // High regs or no regs.
20210 if (Subtarget->isThumb())
20211 return RCPair(0U, &ARM::hGPRRegClass);
20212 break;
20213 case 'r':
20214 if (Subtarget->isThumb1Only())
20215 return RCPair(0U, &ARM::tGPRRegClass);
20216 return RCPair(0U, &ARM::GPRRegClass);
20217 case 'w':
20218 if (VT == MVT::Other)
20219 break;
20220 if (VT == MVT::f16 || VT == MVT::bf16)
20221 return RCPair(0U, &ARM::HPRRegClass);
20222 if (VT == MVT::f32)
20223 return RCPair(0U, &ARM::SPRRegClass);
20224 if (VT.getSizeInBits() == 64)
20225 return RCPair(0U, &ARM::DPRRegClass);
20226 if (VT.getSizeInBits() == 128)
20227 return RCPair(0U, &ARM::QPRRegClass);
20228 break;
20229 case 'x':
20230 if (VT == MVT::Other)
20231 break;
20232 if (VT == MVT::f32)
20233 return RCPair(0U, &ARM::SPR_8RegClass);
20234 if (VT.getSizeInBits() == 64)
20235 return RCPair(0U, &ARM::DPR_8RegClass);
20236 if (VT.getSizeInBits() == 128)
20237 return RCPair(0U, &ARM::QPR_8RegClass);
20238 break;
20239 case 't':
20240 if (VT == MVT::Other)
20241 break;
20242 if (VT == MVT::f16 || VT == MVT::bf16)
20243 return RCPair(0U, &ARM::HPRRegClass);
20244 if (VT == MVT::f32 || VT == MVT::i32)
20245 return RCPair(0U, &ARM::SPRRegClass);
20246 if (VT.getSizeInBits() == 64)
20247 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20248 if (VT.getSizeInBits() == 128)
20249 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20250 break;
20251 }
20252 break;
20253
20254 case 2:
20255 if (Constraint[0] == 'T') {
20256 switch (Constraint[1]) {
20257 default:
20258 break;
20259 case 'e':
20260 return RCPair(0U, &ARM::tGPREvenRegClass);
20261 case 'o':
20262 return RCPair(0U, &ARM::tGPROddRegClass);
20263 }
20264 }
20265 break;
20266
20267 default:
20268 break;
20269 }
20270
20271 if (StringRef("{cc}").equals_insensitive(Constraint))
20272 return std::make_pair(unsigned(ARM::CPSR), &ARM::CCRRegClass);
20273
20274 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20275 }
20276
20277 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
20278 /// vector. If it is invalid, don't add anything to Ops.
LowerAsmOperandForConstraint(SDValue Op,std::string & Constraint,std::vector<SDValue> & Ops,SelectionDAG & DAG) const20279 void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
20280 std::string &Constraint,
20281 std::vector<SDValue>&Ops,
20282 SelectionDAG &DAG) const {
20283 SDValue Result;
20284
20285 // Currently only support length 1 constraints.
20286 if (Constraint.length() != 1) return;
20287
20288 char ConstraintLetter = Constraint[0];
20289 switch (ConstraintLetter) {
20290 default: break;
20291 case 'j':
20292 case 'I': case 'J': case 'K': case 'L':
20293 case 'M': case 'N': case 'O':
20294 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
20295 if (!C)
20296 return;
20297
20298 int64_t CVal64 = C->getSExtValue();
20299 int CVal = (int) CVal64;
20300 // None of these constraints allow values larger than 32 bits. Check
20301 // that the value fits in an int.
20302 if (CVal != CVal64)
20303 return;
20304
20305 switch (ConstraintLetter) {
20306 case 'j':
20307 // Constant suitable for movw, must be between 0 and
20308 // 65535.
20309 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20310 if (CVal >= 0 && CVal <= 65535)
20311 break;
20312 return;
20313 case 'I':
20314 if (Subtarget->isThumb1Only()) {
20315 // This must be a constant between 0 and 255, for ADD
20316 // immediates.
20317 if (CVal >= 0 && CVal <= 255)
20318 break;
20319 } else if (Subtarget->isThumb2()) {
20320 // A constant that can be used as an immediate value in a
20321 // data-processing instruction.
20322 if (ARM_AM::getT2SOImmVal(CVal) != -1)
20323 break;
20324 } else {
20325 // A constant that can be used as an immediate value in a
20326 // data-processing instruction.
20327 if (ARM_AM::getSOImmVal(CVal) != -1)
20328 break;
20329 }
20330 return;
20331
20332 case 'J':
20333 if (Subtarget->isThumb1Only()) {
20334 // This must be a constant between -255 and -1, for negated ADD
20335 // immediates. This can be used in GCC with an "n" modifier that
20336 // prints the negated value, for use with SUB instructions. It is
20337 // not useful otherwise but is implemented for compatibility.
20338 if (CVal >= -255 && CVal <= -1)
20339 break;
20340 } else {
20341 // This must be a constant between -4095 and 4095. It is not clear
20342 // what this constraint is intended for. Implemented for
20343 // compatibility with GCC.
20344 if (CVal >= -4095 && CVal <= 4095)
20345 break;
20346 }
20347 return;
20348
20349 case 'K':
20350 if (Subtarget->isThumb1Only()) {
20351 // A 32-bit value where only one byte has a nonzero value. Exclude
20352 // zero to match GCC. This constraint is used by GCC internally for
20353 // constants that can be loaded with a move/shift combination.
20354 // It is not useful otherwise but is implemented for compatibility.
20355 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(CVal))
20356 break;
20357 } else if (Subtarget->isThumb2()) {
20358 // A constant whose bitwise inverse can be used as an immediate
20359 // value in a data-processing instruction. This can be used in GCC
20360 // with a "B" modifier that prints the inverted value, for use with
20361 // BIC and MVN instructions. It is not useful otherwise but is
20362 // implemented for compatibility.
20363 if (ARM_AM::getT2SOImmVal(~CVal) != -1)
20364 break;
20365 } else {
20366 // A constant whose bitwise inverse can be used as an immediate
20367 // value in a data-processing instruction. This can be used in GCC
20368 // with a "B" modifier that prints the inverted value, for use with
20369 // BIC and MVN instructions. It is not useful otherwise but is
20370 // implemented for compatibility.
20371 if (ARM_AM::getSOImmVal(~CVal) != -1)
20372 break;
20373 }
20374 return;
20375
20376 case 'L':
20377 if (Subtarget->isThumb1Only()) {
20378 // This must be a constant between -7 and 7,
20379 // for 3-operand ADD/SUB immediate instructions.
20380 if (CVal >= -7 && CVal < 7)
20381 break;
20382 } else if (Subtarget->isThumb2()) {
20383 // A constant whose negation can be used as an immediate value in a
20384 // data-processing instruction. This can be used in GCC with an "n"
20385 // modifier that prints the negated value, for use with SUB
20386 // instructions. It is not useful otherwise but is implemented for
20387 // compatibility.
20388 if (ARM_AM::getT2SOImmVal(-CVal) != -1)
20389 break;
20390 } else {
20391 // A constant whose negation can be used as an immediate value in a
20392 // data-processing instruction. This can be used in GCC with an "n"
20393 // modifier that prints the negated value, for use with SUB
20394 // instructions. It is not useful otherwise but is implemented for
20395 // compatibility.
20396 if (ARM_AM::getSOImmVal(-CVal) != -1)
20397 break;
20398 }
20399 return;
20400
20401 case 'M':
20402 if (Subtarget->isThumb1Only()) {
20403 // This must be a multiple of 4 between 0 and 1020, for
20404 // ADD sp + immediate.
20405 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20406 break;
20407 } else {
20408 // A power of two or a constant between 0 and 32. This is used in
20409 // GCC for the shift amount on shifted register operands, but it is
20410 // useful in general for any shift amounts.
20411 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20412 break;
20413 }
20414 return;
20415
20416 case 'N':
20417 if (Subtarget->isThumb1Only()) {
20418 // This must be a constant between 0 and 31, for shift amounts.
20419 if (CVal >= 0 && CVal <= 31)
20420 break;
20421 }
20422 return;
20423
20424 case 'O':
20425 if (Subtarget->isThumb1Only()) {
20426 // This must be a multiple of 4 between -508 and 508, for
20427 // ADD/SUB sp = sp + immediate.
20428 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20429 break;
20430 }
20431 return;
20432 }
20433 Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType());
20434 break;
20435 }
20436
20437 if (Result.getNode()) {
20438 Ops.push_back(Result);
20439 return;
20440 }
20441 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
20442 }
20443
getDivRemLibcall(const SDNode * N,MVT::SimpleValueType SVT)20444 static RTLIB::Libcall getDivRemLibcall(
20445 const SDNode *N, MVT::SimpleValueType SVT) {
20446 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
20447 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
20448 "Unhandled Opcode in getDivRemLibcall");
20449 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
20450 N->getOpcode() == ISD::SREM;
20451 RTLIB::Libcall LC;
20452 switch (SVT) {
20453 default: llvm_unreachable("Unexpected request for libcall!");
20454 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
20455 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
20456 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
20457 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
20458 }
20459 return LC;
20460 }
20461
getDivRemArgList(const SDNode * N,LLVMContext * Context,const ARMSubtarget * Subtarget)20462 static TargetLowering::ArgListTy getDivRemArgList(
20463 const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget) {
20464 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
20465 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
20466 "Unhandled Opcode in getDivRemArgList");
20467 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
20468 N->getOpcode() == ISD::SREM;
20469 TargetLowering::ArgListTy Args;
20470 TargetLowering::ArgListEntry Entry;
20471 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
20472 EVT ArgVT = N->getOperand(i).getValueType();
20473 Type *ArgTy = ArgVT.getTypeForEVT(*Context);
20474 Entry.Node = N->getOperand(i);
20475 Entry.Ty = ArgTy;
20476 Entry.IsSExt = isSigned;
20477 Entry.IsZExt = !isSigned;
20478 Args.push_back(Entry);
20479 }
20480 if (Subtarget->isTargetWindows() && Args.size() >= 2)
20481 std::swap(Args[0], Args[1]);
20482 return Args;
20483 }
20484
LowerDivRem(SDValue Op,SelectionDAG & DAG) const20485 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
20486 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
20487 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
20488 Subtarget->isTargetWindows()) &&
20489 "Register-based DivRem lowering only");
20490 unsigned Opcode = Op->getOpcode();
20491 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
20492 "Invalid opcode for Div/Rem lowering");
20493 bool isSigned = (Opcode == ISD::SDIVREM);
20494 EVT VT = Op->getValueType(0);
20495 SDLoc dl(Op);
20496
20497 if (VT == MVT::i64 && isa<ConstantSDNode>(Op.getOperand(1))) {
20498 SmallVector<SDValue> Result;
20499 if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i32, DAG)) {
20500 SDValue Res0 =
20501 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[0], Result[1]);
20502 SDValue Res1 =
20503 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[2], Result[3]);
20504 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
20505 {Res0, Res1});
20506 }
20507 }
20508
20509 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
20510
20511 // If the target has hardware divide, use divide + multiply + subtract:
20512 // div = a / b
20513 // rem = a - b * div
20514 // return {div, rem}
20515 // This should be lowered into UDIV/SDIV + MLS later on.
20516 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20517 : Subtarget->hasDivideInARMMode();
20518 if (hasDivide && Op->getValueType(0).isSimple() &&
20519 Op->getSimpleValueType(0) == MVT::i32) {
20520 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
20521 const SDValue Dividend = Op->getOperand(0);
20522 const SDValue Divisor = Op->getOperand(1);
20523 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor);
20524 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor);
20525 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
20526
20527 SDValue Values[2] = {Div, Rem};
20528 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values);
20529 }
20530
20531 RTLIB::Libcall LC = getDivRemLibcall(Op.getNode(),
20532 VT.getSimpleVT().SimpleTy);
20533 SDValue InChain = DAG.getEntryNode();
20534
20535 TargetLowering::ArgListTy Args = getDivRemArgList(Op.getNode(),
20536 DAG.getContext(),
20537 Subtarget);
20538
20539 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
20540 getPointerTy(DAG.getDataLayout()));
20541
20542 Type *RetTy = StructType::get(Ty, Ty);
20543
20544 if (Subtarget->isTargetWindows())
20545 InChain = WinDBZCheckDenominator(DAG, Op.getNode(), InChain);
20546
20547 TargetLowering::CallLoweringInfo CLI(DAG);
20548 CLI.setDebugLoc(dl).setChain(InChain)
20549 .setCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
20550 .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned);
20551
20552 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
20553 return CallInfo.first;
20554 }
20555
20556 // Lowers REM using divmod helpers
20557 // see RTABI section 4.2/4.3
LowerREM(SDNode * N,SelectionDAG & DAG) const20558 SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
20559 EVT VT = N->getValueType(0);
20560
20561 if (VT == MVT::i64 && isa<ConstantSDNode>(N->getOperand(1))) {
20562 SmallVector<SDValue> Result;
20563 if (expandDIVREMByConstant(N, Result, MVT::i32, DAG))
20564 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), N->getValueType(0),
20565 Result[0], Result[1]);
20566 }
20567
20568 // Build return types (div and rem)
20569 std::vector<Type*> RetTyParams;
20570 Type *RetTyElement;
20571
20572 switch (VT.getSimpleVT().SimpleTy) {
20573 default: llvm_unreachable("Unexpected request for libcall!");
20574 case MVT::i8: RetTyElement = Type::getInt8Ty(*DAG.getContext()); break;
20575 case MVT::i16: RetTyElement = Type::getInt16Ty(*DAG.getContext()); break;
20576 case MVT::i32: RetTyElement = Type::getInt32Ty(*DAG.getContext()); break;
20577 case MVT::i64: RetTyElement = Type::getInt64Ty(*DAG.getContext()); break;
20578 }
20579
20580 RetTyParams.push_back(RetTyElement);
20581 RetTyParams.push_back(RetTyElement);
20582 ArrayRef<Type*> ret = ArrayRef<Type*>(RetTyParams);
20583 Type *RetTy = StructType::get(*DAG.getContext(), ret);
20584
20585 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT().
20586 SimpleTy);
20587 SDValue InChain = DAG.getEntryNode();
20588 TargetLowering::ArgListTy Args = getDivRemArgList(N, DAG.getContext(),
20589 Subtarget);
20590 bool isSigned = N->getOpcode() == ISD::SREM;
20591 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
20592 getPointerTy(DAG.getDataLayout()));
20593
20594 if (Subtarget->isTargetWindows())
20595 InChain = WinDBZCheckDenominator(DAG, N, InChain);
20596
20597 // Lower call
20598 CallLoweringInfo CLI(DAG);
20599 CLI.setChain(InChain)
20600 .setCallee(CallingConv::ARM_AAPCS, RetTy, Callee, std::move(Args))
20601 .setSExtResult(isSigned).setZExtResult(!isSigned).setDebugLoc(SDLoc(N));
20602 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
20603
20604 // Return second (rem) result operand (first contains div)
20605 SDNode *ResNode = CallResult.first.getNode();
20606 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands");
20607 return ResNode->getOperand(1);
20608 }
20609
20610 SDValue
LowerDYNAMIC_STACKALLOC(SDValue Op,SelectionDAG & DAG) const20611 ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
20612 assert(Subtarget->isTargetWindows() && "unsupported target platform");
20613 SDLoc DL(Op);
20614
20615 // Get the inputs.
20616 SDValue Chain = Op.getOperand(0);
20617 SDValue Size = Op.getOperand(1);
20618
20619 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
20620 "no-stack-arg-probe")) {
20621 MaybeAlign Align =
20622 cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
20623 SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
20624 Chain = SP.getValue(1);
20625 SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size);
20626 if (Align)
20627 SP =
20628 DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0),
20629 DAG.getConstant(-(uint64_t)Align->value(), DL, MVT::i32));
20630 Chain = DAG.getCopyToReg(Chain, DL, ARM::SP, SP);
20631 SDValue Ops[2] = { SP, Chain };
20632 return DAG.getMergeValues(Ops, DL);
20633 }
20634
20635 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
20636 DAG.getConstant(2, DL, MVT::i32));
20637
20638 SDValue Flag;
20639 Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Flag);
20640 Flag = Chain.getValue(1);
20641
20642 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
20643 Chain = DAG.getNode(ARMISD::WIN__CHKSTK, DL, NodeTys, Chain, Flag);
20644
20645 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
20646 Chain = NewSP.getValue(1);
20647
20648 SDValue Ops[2] = { NewSP, Chain };
20649 return DAG.getMergeValues(Ops, DL);
20650 }
20651
LowerFP_EXTEND(SDValue Op,SelectionDAG & DAG) const20652 SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
20653 bool IsStrict = Op->isStrictFPOpcode();
20654 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
20655 const unsigned DstSz = Op.getValueType().getSizeInBits();
20656 const unsigned SrcSz = SrcVal.getValueType().getSizeInBits();
20657 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20658 "Unexpected type for custom-lowering FP_EXTEND");
20659
20660 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
20661 "With both FP DP and 16, any FP conversion is legal!");
20662
20663 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20664 "With FP16, 16 to 32 conversion is legal!");
20665
20666 // Converting from 32 -> 64 is valid if we have FP64.
20667 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20668 // FIXME: Remove this when we have strict fp instruction selection patterns
20669 if (IsStrict) {
20670 SDLoc Loc(Op);
20671 SDValue Result = DAG.getNode(ISD::FP_EXTEND,
20672 Loc, Op.getValueType(), SrcVal);
20673 return DAG.getMergeValues({Result, Op.getOperand(0)}, Loc);
20674 }
20675 return Op;
20676 }
20677
20678 // Either we are converting from 16 -> 64, without FP16 and/or
20679 // FP.double-precision or without Armv8-fp. So we must do it in two
20680 // steps.
20681 // Or we are converting from 32 -> 64 without fp.double-precision or 16 -> 32
20682 // without FP16. So we must do a function call.
20683 SDLoc Loc(Op);
20684 RTLIB::Libcall LC;
20685 MakeLibCallOptions CallOptions;
20686 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
20687 for (unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20688 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20689 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20690 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20691 if (Supported) {
20692 if (IsStrict) {
20693 SrcVal = DAG.getNode(ISD::STRICT_FP_EXTEND, Loc,
20694 {DstVT, MVT::Other}, {Chain, SrcVal});
20695 Chain = SrcVal.getValue(1);
20696 } else {
20697 SrcVal = DAG.getNode(ISD::FP_EXTEND, Loc, DstVT, SrcVal);
20698 }
20699 } else {
20700 LC = RTLIB::getFPEXT(SrcVT, DstVT);
20701 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20702 "Unexpected type for custom-lowering FP_EXTEND");
20703 std::tie(SrcVal, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20704 Loc, Chain);
20705 }
20706 }
20707
20708 return IsStrict ? DAG.getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20709 }
20710
LowerFP_ROUND(SDValue Op,SelectionDAG & DAG) const20711 SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
20712 bool IsStrict = Op->isStrictFPOpcode();
20713
20714 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
20715 EVT SrcVT = SrcVal.getValueType();
20716 EVT DstVT = Op.getValueType();
20717 const unsigned DstSz = Op.getValueType().getSizeInBits();
20718 const unsigned SrcSz = SrcVT.getSizeInBits();
20719 (void)DstSz;
20720 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20721 "Unexpected type for custom-lowering FP_ROUND");
20722
20723 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
20724 "With both FP DP and 16, any FP conversion is legal!");
20725
20726 SDLoc Loc(Op);
20727
20728 // Instruction from 32 -> 16 if hasFP16 is valid
20729 if (SrcSz == 32 && Subtarget->hasFP16())
20730 return Op;
20731
20732 // Lib call from 32 -> 16 / 64 -> [32, 16]
20733 RTLIB::Libcall LC = RTLIB::getFPROUND(SrcVT, DstVT);
20734 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20735 "Unexpected type for custom-lowering FP_ROUND");
20736 MakeLibCallOptions CallOptions;
20737 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
20738 SDValue Result;
20739 std::tie(Result, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20740 Loc, Chain);
20741 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
20742 }
20743
20744 bool
isOffsetFoldingLegal(const GlobalAddressSDNode * GA) const20745 ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
20746 // The ARM target isn't yet aware of offsets.
20747 return false;
20748 }
20749
isBitFieldInvertedMask(unsigned v)20750 bool ARM::isBitFieldInvertedMask(unsigned v) {
20751 if (v == 0xffffffff)
20752 return false;
20753
20754 // there can be 1's on either or both "outsides", all the "inside"
20755 // bits must be 0's
20756 return isShiftedMask_32(~v);
20757 }
20758
20759 /// isFPImmLegal - Returns true if the target can instruction select the
20760 /// specified FP immediate natively. If false, the legalizer will
20761 /// materialize the FP immediate as a load from a constant pool.
isFPImmLegal(const APFloat & Imm,EVT VT,bool ForCodeSize) const20762 bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
20763 bool ForCodeSize) const {
20764 if (!Subtarget->hasVFP3Base())
20765 return false;
20766 if (VT == MVT::f16 && Subtarget->hasFullFP16())
20767 return ARM_AM::getFP16Imm(Imm) != -1;
20768 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
20769 ARM_AM::getFP32FP16Imm(Imm) != -1)
20770 return true;
20771 if (VT == MVT::f32)
20772 return ARM_AM::getFP32Imm(Imm) != -1;
20773 if (VT == MVT::f64 && Subtarget->hasFP64())
20774 return ARM_AM::getFP64Imm(Imm) != -1;
20775 return false;
20776 }
20777
20778 /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
20779 /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
20780 /// specified in the intrinsic calls.
getTgtMemIntrinsic(IntrinsicInfo & Info,const CallInst & I,MachineFunction & MF,unsigned Intrinsic) const20781 bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
20782 const CallInst &I,
20783 MachineFunction &MF,
20784 unsigned Intrinsic) const {
20785 switch (Intrinsic) {
20786 case Intrinsic::arm_neon_vld1:
20787 case Intrinsic::arm_neon_vld2:
20788 case Intrinsic::arm_neon_vld3:
20789 case Intrinsic::arm_neon_vld4:
20790 case Intrinsic::arm_neon_vld2lane:
20791 case Intrinsic::arm_neon_vld3lane:
20792 case Intrinsic::arm_neon_vld4lane:
20793 case Intrinsic::arm_neon_vld2dup:
20794 case Intrinsic::arm_neon_vld3dup:
20795 case Intrinsic::arm_neon_vld4dup: {
20796 Info.opc = ISD::INTRINSIC_W_CHAIN;
20797 // Conservatively set memVT to the entire set of vectors loaded.
20798 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
20799 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
20800 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
20801 Info.ptrVal = I.getArgOperand(0);
20802 Info.offset = 0;
20803 Value *AlignArg = I.getArgOperand(I.arg_size() - 1);
20804 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
20805 // volatile loads with NEON intrinsics not supported
20806 Info.flags = MachineMemOperand::MOLoad;
20807 return true;
20808 }
20809 case Intrinsic::arm_neon_vld1x2:
20810 case Intrinsic::arm_neon_vld1x3:
20811 case Intrinsic::arm_neon_vld1x4: {
20812 Info.opc = ISD::INTRINSIC_W_CHAIN;
20813 // Conservatively set memVT to the entire set of vectors loaded.
20814 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
20815 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
20816 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
20817 Info.ptrVal = I.getArgOperand(I.arg_size() - 1);
20818 Info.offset = 0;
20819 Info.align.reset();
20820 // volatile loads with NEON intrinsics not supported
20821 Info.flags = MachineMemOperand::MOLoad;
20822 return true;
20823 }
20824 case Intrinsic::arm_neon_vst1:
20825 case Intrinsic::arm_neon_vst2:
20826 case Intrinsic::arm_neon_vst3:
20827 case Intrinsic::arm_neon_vst4:
20828 case Intrinsic::arm_neon_vst2lane:
20829 case Intrinsic::arm_neon_vst3lane:
20830 case Intrinsic::arm_neon_vst4lane: {
20831 Info.opc = ISD::INTRINSIC_VOID;
20832 // Conservatively set memVT to the entire set of vectors stored.
20833 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
20834 unsigned NumElts = 0;
20835 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
20836 Type *ArgTy = I.getArgOperand(ArgI)->getType();
20837 if (!ArgTy->isVectorTy())
20838 break;
20839 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
20840 }
20841 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
20842 Info.ptrVal = I.getArgOperand(0);
20843 Info.offset = 0;
20844 Value *AlignArg = I.getArgOperand(I.arg_size() - 1);
20845 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
20846 // volatile stores with NEON intrinsics not supported
20847 Info.flags = MachineMemOperand::MOStore;
20848 return true;
20849 }
20850 case Intrinsic::arm_neon_vst1x2:
20851 case Intrinsic::arm_neon_vst1x3:
20852 case Intrinsic::arm_neon_vst1x4: {
20853 Info.opc = ISD::INTRINSIC_VOID;
20854 // Conservatively set memVT to the entire set of vectors stored.
20855 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
20856 unsigned NumElts = 0;
20857 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
20858 Type *ArgTy = I.getArgOperand(ArgI)->getType();
20859 if (!ArgTy->isVectorTy())
20860 break;
20861 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
20862 }
20863 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
20864 Info.ptrVal = I.getArgOperand(0);
20865 Info.offset = 0;
20866 Info.align.reset();
20867 // volatile stores with NEON intrinsics not supported
20868 Info.flags = MachineMemOperand::MOStore;
20869 return true;
20870 }
20871 case Intrinsic::arm_mve_vld2q:
20872 case Intrinsic::arm_mve_vld4q: {
20873 Info.opc = ISD::INTRINSIC_W_CHAIN;
20874 // Conservatively set memVT to the entire set of vectors loaded.
20875 Type *VecTy = cast<StructType>(I.getType())->getElementType(1);
20876 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
20877 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2);
20878 Info.ptrVal = I.getArgOperand(0);
20879 Info.offset = 0;
20880 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
20881 // volatile loads with MVE intrinsics not supported
20882 Info.flags = MachineMemOperand::MOLoad;
20883 return true;
20884 }
20885 case Intrinsic::arm_mve_vst2q:
20886 case Intrinsic::arm_mve_vst4q: {
20887 Info.opc = ISD::INTRINSIC_VOID;
20888 // Conservatively set memVT to the entire set of vectors stored.
20889 Type *VecTy = I.getArgOperand(1)->getType();
20890 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
20891 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2);
20892 Info.ptrVal = I.getArgOperand(0);
20893 Info.offset = 0;
20894 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
20895 // volatile stores with MVE intrinsics not supported
20896 Info.flags = MachineMemOperand::MOStore;
20897 return true;
20898 }
20899 case Intrinsic::arm_mve_vldr_gather_base:
20900 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
20901 Info.opc = ISD::INTRINSIC_W_CHAIN;
20902 Info.ptrVal = nullptr;
20903 Info.memVT = MVT::getVT(I.getType());
20904 Info.align = Align(1);
20905 Info.flags |= MachineMemOperand::MOLoad;
20906 return true;
20907 }
20908 case Intrinsic::arm_mve_vldr_gather_base_wb:
20909 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
20910 Info.opc = ISD::INTRINSIC_W_CHAIN;
20911 Info.ptrVal = nullptr;
20912 Info.memVT = MVT::getVT(I.getType()->getContainedType(0));
20913 Info.align = Align(1);
20914 Info.flags |= MachineMemOperand::MOLoad;
20915 return true;
20916 }
20917 case Intrinsic::arm_mve_vldr_gather_offset:
20918 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
20919 Info.opc = ISD::INTRINSIC_W_CHAIN;
20920 Info.ptrVal = nullptr;
20921 MVT DataVT = MVT::getVT(I.getType());
20922 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
20923 Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize),
20924 DataVT.getVectorNumElements());
20925 Info.align = Align(1);
20926 Info.flags |= MachineMemOperand::MOLoad;
20927 return true;
20928 }
20929 case Intrinsic::arm_mve_vstr_scatter_base:
20930 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
20931 Info.opc = ISD::INTRINSIC_VOID;
20932 Info.ptrVal = nullptr;
20933 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType());
20934 Info.align = Align(1);
20935 Info.flags |= MachineMemOperand::MOStore;
20936 return true;
20937 }
20938 case Intrinsic::arm_mve_vstr_scatter_base_wb:
20939 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
20940 Info.opc = ISD::INTRINSIC_W_CHAIN;
20941 Info.ptrVal = nullptr;
20942 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType());
20943 Info.align = Align(1);
20944 Info.flags |= MachineMemOperand::MOStore;
20945 return true;
20946 }
20947 case Intrinsic::arm_mve_vstr_scatter_offset:
20948 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
20949 Info.opc = ISD::INTRINSIC_VOID;
20950 Info.ptrVal = nullptr;
20951 MVT DataVT = MVT::getVT(I.getArgOperand(2)->getType());
20952 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
20953 Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize),
20954 DataVT.getVectorNumElements());
20955 Info.align = Align(1);
20956 Info.flags |= MachineMemOperand::MOStore;
20957 return true;
20958 }
20959 case Intrinsic::arm_ldaex:
20960 case Intrinsic::arm_ldrex: {
20961 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
20962 Type *ValTy = I.getParamElementType(0);
20963 Info.opc = ISD::INTRINSIC_W_CHAIN;
20964 Info.memVT = MVT::getVT(ValTy);
20965 Info.ptrVal = I.getArgOperand(0);
20966 Info.offset = 0;
20967 Info.align = DL.getABITypeAlign(ValTy);
20968 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
20969 return true;
20970 }
20971 case Intrinsic::arm_stlex:
20972 case Intrinsic::arm_strex: {
20973 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
20974 Type *ValTy = I.getParamElementType(1);
20975 Info.opc = ISD::INTRINSIC_W_CHAIN;
20976 Info.memVT = MVT::getVT(ValTy);
20977 Info.ptrVal = I.getArgOperand(1);
20978 Info.offset = 0;
20979 Info.align = DL.getABITypeAlign(ValTy);
20980 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
20981 return true;
20982 }
20983 case Intrinsic::arm_stlexd:
20984 case Intrinsic::arm_strexd:
20985 Info.opc = ISD::INTRINSIC_W_CHAIN;
20986 Info.memVT = MVT::i64;
20987 Info.ptrVal = I.getArgOperand(2);
20988 Info.offset = 0;
20989 Info.align = Align(8);
20990 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
20991 return true;
20992
20993 case Intrinsic::arm_ldaexd:
20994 case Intrinsic::arm_ldrexd:
20995 Info.opc = ISD::INTRINSIC_W_CHAIN;
20996 Info.memVT = MVT::i64;
20997 Info.ptrVal = I.getArgOperand(0);
20998 Info.offset = 0;
20999 Info.align = Align(8);
21000 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
21001 return true;
21002
21003 default:
21004 break;
21005 }
21006
21007 return false;
21008 }
21009
21010 /// Returns true if it is beneficial to convert a load of a constant
21011 /// to just the constant itself.
shouldConvertConstantLoadToIntImm(const APInt & Imm,Type * Ty) const21012 bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
21013 Type *Ty) const {
21014 assert(Ty->isIntegerTy());
21015
21016 unsigned Bits = Ty->getPrimitiveSizeInBits();
21017 if (Bits == 0 || Bits > 32)
21018 return false;
21019 return true;
21020 }
21021
isExtractSubvectorCheap(EVT ResVT,EVT SrcVT,unsigned Index) const21022 bool ARMTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
21023 unsigned Index) const {
21024 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
21025 return false;
21026
21027 return (Index == 0 || Index == ResVT.getVectorNumElements());
21028 }
21029
makeDMB(IRBuilderBase & Builder,ARM_MB::MemBOpt Domain) const21030 Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
21031 ARM_MB::MemBOpt Domain) const {
21032 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21033
21034 // First, if the target has no DMB, see what fallback we can use.
21035 if (!Subtarget->hasDataBarrier()) {
21036 // Some ARMv6 cpus can support data barriers with an mcr instruction.
21037 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
21038 // here.
21039 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21040 Function *MCR = Intrinsic::getDeclaration(M, Intrinsic::arm_mcr);
21041 Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
21042 Builder.getInt32(0), Builder.getInt32(7),
21043 Builder.getInt32(10), Builder.getInt32(5)};
21044 return Builder.CreateCall(MCR, args);
21045 } else {
21046 // Instead of using barriers, atomic accesses on these subtargets use
21047 // libcalls.
21048 llvm_unreachable("makeDMB on a target so old that it has no barriers");
21049 }
21050 } else {
21051 Function *DMB = Intrinsic::getDeclaration(M, Intrinsic::arm_dmb);
21052 // Only a full system barrier exists in the M-class architectures.
21053 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
21054 Constant *CDomain = Builder.getInt32(Domain);
21055 return Builder.CreateCall(DMB, CDomain);
21056 }
21057 }
21058
21059 // Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
emitLeadingFence(IRBuilderBase & Builder,Instruction * Inst,AtomicOrdering Ord) const21060 Instruction *ARMTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
21061 Instruction *Inst,
21062 AtomicOrdering Ord) const {
21063 switch (Ord) {
21064 case AtomicOrdering::NotAtomic:
21065 case AtomicOrdering::Unordered:
21066 llvm_unreachable("Invalid fence: unordered/non-atomic");
21067 case AtomicOrdering::Monotonic:
21068 case AtomicOrdering::Acquire:
21069 return nullptr; // Nothing to do
21070 case AtomicOrdering::SequentiallyConsistent:
21071 if (!Inst->hasAtomicStore())
21072 return nullptr; // Nothing to do
21073 [[fallthrough]];
21074 case AtomicOrdering::Release:
21075 case AtomicOrdering::AcquireRelease:
21076 if (Subtarget->preferISHSTBarriers())
21077 return makeDMB(Builder, ARM_MB::ISHST);
21078 // FIXME: add a comment with a link to documentation justifying this.
21079 else
21080 return makeDMB(Builder, ARM_MB::ISH);
21081 }
21082 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
21083 }
21084
emitTrailingFence(IRBuilderBase & Builder,Instruction * Inst,AtomicOrdering Ord) const21085 Instruction *ARMTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
21086 Instruction *Inst,
21087 AtomicOrdering Ord) const {
21088 switch (Ord) {
21089 case AtomicOrdering::NotAtomic:
21090 case AtomicOrdering::Unordered:
21091 llvm_unreachable("Invalid fence: unordered/not-atomic");
21092 case AtomicOrdering::Monotonic:
21093 case AtomicOrdering::Release:
21094 return nullptr; // Nothing to do
21095 case AtomicOrdering::Acquire:
21096 case AtomicOrdering::AcquireRelease:
21097 case AtomicOrdering::SequentiallyConsistent:
21098 return makeDMB(Builder, ARM_MB::ISH);
21099 }
21100 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
21101 }
21102
21103 // Loads and stores less than 64-bits are already atomic; ones above that
21104 // are doomed anyway, so defer to the default libcall and blame the OS when
21105 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21106 // anything for those.
21107 TargetLoweringBase::AtomicExpansionKind
shouldExpandAtomicStoreInIR(StoreInst * SI) const21108 ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
21109 bool has64BitAtomicStore;
21110 if (Subtarget->isMClass())
21111 has64BitAtomicStore = false;
21112 else if (Subtarget->isThumb())
21113 has64BitAtomicStore = Subtarget->hasV7Ops();
21114 else
21115 has64BitAtomicStore = Subtarget->hasV6Ops();
21116
21117 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21118 return Size == 64 && has64BitAtomicStore ? AtomicExpansionKind::Expand
21119 : AtomicExpansionKind::None;
21120 }
21121
21122 // Loads and stores less than 64-bits are already atomic; ones above that
21123 // are doomed anyway, so defer to the default libcall and blame the OS when
21124 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21125 // anything for those.
21126 // FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
21127 // guarantee, see DDI0406C ARM architecture reference manual,
21128 // sections A8.8.72-74 LDRD)
21129 TargetLowering::AtomicExpansionKind
shouldExpandAtomicLoadInIR(LoadInst * LI) const21130 ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
21131 bool has64BitAtomicLoad;
21132 if (Subtarget->isMClass())
21133 has64BitAtomicLoad = false;
21134 else if (Subtarget->isThumb())
21135 has64BitAtomicLoad = Subtarget->hasV7Ops();
21136 else
21137 has64BitAtomicLoad = Subtarget->hasV6Ops();
21138
21139 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
21140 return (Size == 64 && has64BitAtomicLoad) ? AtomicExpansionKind::LLOnly
21141 : AtomicExpansionKind::None;
21142 }
21143
21144 // For the real atomic operations, we have ldrex/strex up to 32 bits,
21145 // and up to 64 bits on the non-M profiles
21146 TargetLowering::AtomicExpansionKind
shouldExpandAtomicRMWInIR(AtomicRMWInst * AI) const21147 ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
21148 if (AI->isFloatingPointOperation())
21149 return AtomicExpansionKind::CmpXChg;
21150
21151 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21152 bool hasAtomicRMW;
21153 if (Subtarget->isMClass())
21154 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21155 else if (Subtarget->isThumb())
21156 hasAtomicRMW = Subtarget->hasV7Ops();
21157 else
21158 hasAtomicRMW = Subtarget->hasV6Ops();
21159 if (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) {
21160 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21161 // implement atomicrmw without spilling. If the target address is also on
21162 // the stack and close enough to the spill slot, this can lead to a
21163 // situation where the monitor always gets cleared and the atomic operation
21164 // can never succeed. So at -O0 lower this operation to a CAS loop.
21165 if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
21166 return AtomicExpansionKind::CmpXChg;
21167 return AtomicExpansionKind::LLSC;
21168 }
21169 return AtomicExpansionKind::None;
21170 }
21171
21172 // Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32
21173 // bits, and up to 64 bits on the non-M profiles.
21174 TargetLowering::AtomicExpansionKind
shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst * AI) const21175 ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
21176 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21177 // implement cmpxchg without spilling. If the address being exchanged is also
21178 // on the stack and close enough to the spill slot, this can lead to a
21179 // situation where the monitor always gets cleared and the atomic operation
21180 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
21181 unsigned Size = AI->getOperand(1)->getType()->getPrimitiveSizeInBits();
21182 bool HasAtomicCmpXchg;
21183 if (Subtarget->isMClass())
21184 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21185 else if (Subtarget->isThumb())
21186 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21187 else
21188 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21189 if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg &&
21190 Size <= (Subtarget->isMClass() ? 32U : 64U))
21191 return AtomicExpansionKind::LLSC;
21192 return AtomicExpansionKind::None;
21193 }
21194
shouldInsertFencesForAtomic(const Instruction * I) const21195 bool ARMTargetLowering::shouldInsertFencesForAtomic(
21196 const Instruction *I) const {
21197 return InsertFencesForAtomic;
21198 }
21199
useLoadStackGuardNode() const21200 bool ARMTargetLowering::useLoadStackGuardNode() const {
21201 // ROPI/RWPI are not supported currently.
21202 return !Subtarget->isROPI() && !Subtarget->isRWPI();
21203 }
21204
insertSSPDeclarations(Module & M) const21205 void ARMTargetLowering::insertSSPDeclarations(Module &M) const {
21206 if (!Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
21207 return TargetLowering::insertSSPDeclarations(M);
21208
21209 // MSVC CRT has a global variable holding security cookie.
21210 M.getOrInsertGlobal("__security_cookie",
21211 Type::getInt8PtrTy(M.getContext()));
21212
21213 // MSVC CRT has a function to validate security cookie.
21214 FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
21215 "__security_check_cookie", Type::getVoidTy(M.getContext()),
21216 Type::getInt8PtrTy(M.getContext()));
21217 if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee()))
21218 F->addParamAttr(0, Attribute::AttrKind::InReg);
21219 }
21220
getSDagStackGuard(const Module & M) const21221 Value *ARMTargetLowering::getSDagStackGuard(const Module &M) const {
21222 // MSVC CRT has a global variable holding security cookie.
21223 if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
21224 return M.getGlobalVariable("__security_cookie");
21225 return TargetLowering::getSDagStackGuard(M);
21226 }
21227
getSSPStackGuardCheck(const Module & M) const21228 Function *ARMTargetLowering::getSSPStackGuardCheck(const Module &M) const {
21229 // MSVC CRT has a function to validate security cookie.
21230 if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
21231 return M.getFunction("__security_check_cookie");
21232 return TargetLowering::getSSPStackGuardCheck(M);
21233 }
21234
canCombineStoreAndExtract(Type * VectorTy,Value * Idx,unsigned & Cost) const21235 bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
21236 unsigned &Cost) const {
21237 // If we do not have NEON, vector types are not natively supported.
21238 if (!Subtarget->hasNEON())
21239 return false;
21240
21241 // Floating point values and vector values map to the same register file.
21242 // Therefore, although we could do a store extract of a vector type, this is
21243 // better to leave at float as we have more freedom in the addressing mode for
21244 // those.
21245 if (VectorTy->isFPOrFPVectorTy())
21246 return false;
21247
21248 // If the index is unknown at compile time, this is very expensive to lower
21249 // and it is not possible to combine the store with the extract.
21250 if (!isa<ConstantInt>(Idx))
21251 return false;
21252
21253 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type");
21254 unsigned BitWidth = VectorTy->getPrimitiveSizeInBits().getFixedValue();
21255 // We can do a store + vector extract on any vector that fits perfectly in a D
21256 // or Q register.
21257 if (BitWidth == 64 || BitWidth == 128) {
21258 Cost = 0;
21259 return true;
21260 }
21261 return false;
21262 }
21263
isCheapToSpeculateCttz(Type * Ty) const21264 bool ARMTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
21265 return Subtarget->hasV6T2Ops();
21266 }
21267
isCheapToSpeculateCtlz(Type * Ty) const21268 bool ARMTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
21269 return Subtarget->hasV6T2Ops();
21270 }
21271
isMaskAndCmp0FoldingBeneficial(const Instruction & AndI) const21272 bool ARMTargetLowering::isMaskAndCmp0FoldingBeneficial(
21273 const Instruction &AndI) const {
21274 if (!Subtarget->hasV7Ops())
21275 return false;
21276
21277 // Sink the `and` instruction only if the mask would fit into a modified
21278 // immediate operand.
21279 ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
21280 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21281 return false;
21282 auto MaskVal = unsigned(Mask->getValue().getZExtValue());
21283 return (Subtarget->isThumb2() ? ARM_AM::getT2SOImmVal(MaskVal)
21284 : ARM_AM::getSOImmVal(MaskVal)) != -1;
21285 }
21286
21287 TargetLowering::ShiftLegalizationStrategy
preferredShiftLegalizationStrategy(SelectionDAG & DAG,SDNode * N,unsigned ExpansionFactor) const21288 ARMTargetLowering::preferredShiftLegalizationStrategy(
21289 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
21290 if (Subtarget->hasMinSize() && !Subtarget->isTargetWindows())
21291 return ShiftLegalizationStrategy::LowerToLibcall;
21292 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
21293 ExpansionFactor);
21294 }
21295
emitLoadLinked(IRBuilderBase & Builder,Type * ValueTy,Value * Addr,AtomicOrdering Ord) const21296 Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
21297 Value *Addr,
21298 AtomicOrdering Ord) const {
21299 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21300 bool IsAcquire = isAcquireOrStronger(Ord);
21301
21302 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
21303 // intrinsic must return {i32, i32} and we have to recombine them into a
21304 // single i64 here.
21305 if (ValueTy->getPrimitiveSizeInBits() == 64) {
21306 Intrinsic::ID Int =
21307 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21308 Function *Ldrex = Intrinsic::getDeclaration(M, Int);
21309
21310 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
21311 Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi");
21312
21313 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
21314 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
21315 if (!Subtarget->isLittle())
21316 std::swap (Lo, Hi);
21317 Lo = Builder.CreateZExt(Lo, ValueTy, "lo64");
21318 Hi = Builder.CreateZExt(Hi, ValueTy, "hi64");
21319 return Builder.CreateOr(
21320 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValueTy, 32)), "val64");
21321 }
21322
21323 Type *Tys[] = { Addr->getType() };
21324 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21325 Function *Ldrex = Intrinsic::getDeclaration(M, Int, Tys);
21326 CallInst *CI = Builder.CreateCall(Ldrex, Addr);
21327
21328 CI->addParamAttr(
21329 0, Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21330 return Builder.CreateTruncOrBitCast(CI, ValueTy);
21331 }
21332
emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase & Builder) const21333 void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
21334 IRBuilderBase &Builder) const {
21335 if (!Subtarget->hasV7Ops())
21336 return;
21337 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21338 Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::arm_clrex));
21339 }
21340
emitStoreConditional(IRBuilderBase & Builder,Value * Val,Value * Addr,AtomicOrdering Ord) const21341 Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
21342 Value *Val, Value *Addr,
21343 AtomicOrdering Ord) const {
21344 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21345 bool IsRelease = isReleaseOrStronger(Ord);
21346
21347 // Since the intrinsics must have legal type, the i64 intrinsics take two
21348 // parameters: "i32, i32". We must marshal Val into the appropriate form
21349 // before the call.
21350 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
21351 Intrinsic::ID Int =
21352 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21353 Function *Strex = Intrinsic::getDeclaration(M, Int);
21354 Type *Int32Ty = Type::getInt32Ty(M->getContext());
21355
21356 Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo");
21357 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi");
21358 if (!Subtarget->isLittle())
21359 std::swap(Lo, Hi);
21360 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
21361 return Builder.CreateCall(Strex, {Lo, Hi, Addr});
21362 }
21363
21364 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21365 Type *Tys[] = { Addr->getType() };
21366 Function *Strex = Intrinsic::getDeclaration(M, Int, Tys);
21367
21368 CallInst *CI = Builder.CreateCall(
21369 Strex, {Builder.CreateZExtOrBitCast(
21370 Val, Strex->getFunctionType()->getParamType(0)),
21371 Addr});
21372 CI->addParamAttr(1, Attribute::get(M->getContext(), Attribute::ElementType,
21373 Val->getType()));
21374 return CI;
21375 }
21376
21377
alignLoopsWithOptSize() const21378 bool ARMTargetLowering::alignLoopsWithOptSize() const {
21379 return Subtarget->isMClass();
21380 }
21381
21382 /// A helper function for determining the number of interleaved accesses we
21383 /// will generate when lowering accesses of the given type.
21384 unsigned
getNumInterleavedAccesses(VectorType * VecTy,const DataLayout & DL) const21385 ARMTargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
21386 const DataLayout &DL) const {
21387 return (DL.getTypeSizeInBits(VecTy) + 127) / 128;
21388 }
21389
isLegalInterleavedAccessType(unsigned Factor,FixedVectorType * VecTy,Align Alignment,const DataLayout & DL) const21390 bool ARMTargetLowering::isLegalInterleavedAccessType(
21391 unsigned Factor, FixedVectorType *VecTy, Align Alignment,
21392 const DataLayout &DL) const {
21393
21394 unsigned VecSize = DL.getTypeSizeInBits(VecTy);
21395 unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
21396
21397 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21398 return false;
21399
21400 // Ensure the vector doesn't have f16 elements. Even though we could do an
21401 // i16 vldN, we can't hold the f16 vectors and will end up converting via
21402 // f32.
21403 if (Subtarget->hasNEON() && VecTy->getElementType()->isHalfTy())
21404 return false;
21405 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21406 return false;
21407
21408 // Ensure the number of vector elements is greater than 1.
21409 if (VecTy->getNumElements() < 2)
21410 return false;
21411
21412 // Ensure the element type is legal.
21413 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21414 return false;
21415 // And the alignment if high enough under MVE.
21416 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21417 return false;
21418
21419 // Ensure the total vector size is 64 or a multiple of 128. Types larger than
21420 // 128 will be split into multiple interleaved accesses.
21421 if (Subtarget->hasNEON() && VecSize == 64)
21422 return true;
21423 return VecSize % 128 == 0;
21424 }
21425
getMaxSupportedInterleaveFactor() const21426 unsigned ARMTargetLowering::getMaxSupportedInterleaveFactor() const {
21427 if (Subtarget->hasNEON())
21428 return 4;
21429 if (Subtarget->hasMVEIntegerOps())
21430 return MVEMaxSupportedInterleaveFactor;
21431 return TargetLoweringBase::getMaxSupportedInterleaveFactor();
21432 }
21433
21434 /// Lower an interleaved load into a vldN intrinsic.
21435 ///
21436 /// E.g. Lower an interleaved load (Factor = 2):
21437 /// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
21438 /// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
21439 /// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
21440 ///
21441 /// Into:
21442 /// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
21443 /// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
21444 /// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
lowerInterleavedLoad(LoadInst * LI,ArrayRef<ShuffleVectorInst * > Shuffles,ArrayRef<unsigned> Indices,unsigned Factor) const21445 bool ARMTargetLowering::lowerInterleavedLoad(
21446 LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
21447 ArrayRef<unsigned> Indices, unsigned Factor) const {
21448 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
21449 "Invalid interleave factor");
21450 assert(!Shuffles.empty() && "Empty shufflevector input");
21451 assert(Shuffles.size() == Indices.size() &&
21452 "Unmatched number of shufflevectors and indices");
21453
21454 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->getType());
21455 Type *EltTy = VecTy->getElementType();
21456
21457 const DataLayout &DL = LI->getModule()->getDataLayout();
21458 Align Alignment = LI->getAlign();
21459
21460 // Skip if we do not have NEON and skip illegal vector types. We can
21461 // "legalize" wide vector types into multiple interleaved accesses as long as
21462 // the vector types are divisible by 128.
21463 if (!isLegalInterleavedAccessType(Factor, VecTy, Alignment, DL))
21464 return false;
21465
21466 unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
21467
21468 // A pointer vector can not be the return type of the ldN intrinsics. Need to
21469 // load integer vectors first and then convert to pointer vectors.
21470 if (EltTy->isPointerTy())
21471 VecTy = FixedVectorType::get(DL.getIntPtrType(EltTy), VecTy);
21472
21473 IRBuilder<> Builder(LI);
21474
21475 // The base address of the load.
21476 Value *BaseAddr = LI->getPointerOperand();
21477
21478 if (NumLoads > 1) {
21479 // If we're going to generate more than one load, reset the sub-vector type
21480 // to something legal.
21481 VecTy = FixedVectorType::get(VecTy->getElementType(),
21482 VecTy->getNumElements() / NumLoads);
21483
21484 // We will compute the pointer operand of each load from the original base
21485 // address using GEPs. Cast the base address to a pointer to the scalar
21486 // element type.
21487 BaseAddr = Builder.CreateBitCast(
21488 BaseAddr,
21489 VecTy->getElementType()->getPointerTo(LI->getPointerAddressSpace()));
21490 }
21491
21492 assert(isTypeLegal(EVT::getEVT(VecTy)) && "Illegal vldN vector type!");
21493
21494 auto createLoadIntrinsic = [&](Value *BaseAddr) {
21495 if (Subtarget->hasNEON()) {
21496 Type *Int8Ptr = Builder.getInt8PtrTy(LI->getPointerAddressSpace());
21497 Type *Tys[] = {VecTy, Int8Ptr};
21498 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21499 Intrinsic::arm_neon_vld3,
21500 Intrinsic::arm_neon_vld4};
21501 Function *VldnFunc =
21502 Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
21503
21504 SmallVector<Value *, 2> Ops;
21505 Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr));
21506 Ops.push_back(Builder.getInt32(LI->getAlign().value()));
21507
21508 return Builder.CreateCall(VldnFunc, Ops, "vldN");
21509 } else {
21510 assert((Factor == 2 || Factor == 4) &&
21511 "expected interleave factor of 2 or 4 for MVE");
21512 Intrinsic::ID LoadInts =
21513 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21514 Type *VecEltTy =
21515 VecTy->getElementType()->getPointerTo(LI->getPointerAddressSpace());
21516 Type *Tys[] = {VecTy, VecEltTy};
21517 Function *VldnFunc =
21518 Intrinsic::getDeclaration(LI->getModule(), LoadInts, Tys);
21519
21520 SmallVector<Value *, 2> Ops;
21521 Ops.push_back(Builder.CreateBitCast(BaseAddr, VecEltTy));
21522 return Builder.CreateCall(VldnFunc, Ops, "vldN");
21523 }
21524 };
21525
21526 // Holds sub-vectors extracted from the load intrinsic return values. The
21527 // sub-vectors are associated with the shufflevector instructions they will
21528 // replace.
21529 DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
21530
21531 for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21532 // If we're generating more than one load, compute the base address of
21533 // subsequent loads as an offset from the previous.
21534 if (LoadCount > 0)
21535 BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
21536 VecTy->getNumElements() * Factor);
21537
21538 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21539
21540 // Replace uses of each shufflevector with the corresponding vector loaded
21541 // by ldN.
21542 for (unsigned i = 0; i < Shuffles.size(); i++) {
21543 ShuffleVectorInst *SV = Shuffles[i];
21544 unsigned Index = Indices[i];
21545
21546 Value *SubVec = Builder.CreateExtractValue(VldN, Index);
21547
21548 // Convert the integer vector to pointer vector if the element is pointer.
21549 if (EltTy->isPointerTy())
21550 SubVec = Builder.CreateIntToPtr(
21551 SubVec,
21552 FixedVectorType::get(SV->getType()->getElementType(), VecTy));
21553
21554 SubVecs[SV].push_back(SubVec);
21555 }
21556 }
21557
21558 // Replace uses of the shufflevector instructions with the sub-vectors
21559 // returned by the load intrinsic. If a shufflevector instruction is
21560 // associated with more than one sub-vector, those sub-vectors will be
21561 // concatenated into a single wide vector.
21562 for (ShuffleVectorInst *SVI : Shuffles) {
21563 auto &SubVec = SubVecs[SVI];
21564 auto *WideVec =
21565 SubVec.size() > 1 ? concatenateVectors(Builder, SubVec) : SubVec[0];
21566 SVI->replaceAllUsesWith(WideVec);
21567 }
21568
21569 return true;
21570 }
21571
21572 /// Lower an interleaved store into a vstN intrinsic.
21573 ///
21574 /// E.g. Lower an interleaved store (Factor = 3):
21575 /// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
21576 /// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
21577 /// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
21578 ///
21579 /// Into:
21580 /// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
21581 /// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
21582 /// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
21583 /// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
21584 ///
21585 /// Note that the new shufflevectors will be removed and we'll only generate one
21586 /// vst3 instruction in CodeGen.
21587 ///
21588 /// Example for a more general valid mask (Factor 3). Lower:
21589 /// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
21590 /// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
21591 /// store <12 x i32> %i.vec, <12 x i32>* %ptr
21592 ///
21593 /// Into:
21594 /// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
21595 /// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
21596 /// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
21597 /// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
lowerInterleavedStore(StoreInst * SI,ShuffleVectorInst * SVI,unsigned Factor) const21598 bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
21599 ShuffleVectorInst *SVI,
21600 unsigned Factor) const {
21601 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
21602 "Invalid interleave factor");
21603
21604 auto *VecTy = cast<FixedVectorType>(SVI->getType());
21605 assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
21606
21607 unsigned LaneLen = VecTy->getNumElements() / Factor;
21608 Type *EltTy = VecTy->getElementType();
21609 auto *SubVecTy = FixedVectorType::get(EltTy, LaneLen);
21610
21611 const DataLayout &DL = SI->getModule()->getDataLayout();
21612 Align Alignment = SI->getAlign();
21613
21614 // Skip if we do not have NEON and skip illegal vector types. We can
21615 // "legalize" wide vector types into multiple interleaved accesses as long as
21616 // the vector types are divisible by 128.
21617 if (!isLegalInterleavedAccessType(Factor, SubVecTy, Alignment, DL))
21618 return false;
21619
21620 unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL);
21621
21622 Value *Op0 = SVI->getOperand(0);
21623 Value *Op1 = SVI->getOperand(1);
21624 IRBuilder<> Builder(SI);
21625
21626 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
21627 // vectors to integer vectors.
21628 if (EltTy->isPointerTy()) {
21629 Type *IntTy = DL.getIntPtrType(EltTy);
21630
21631 // Convert to the corresponding integer vector.
21632 auto *IntVecTy =
21633 FixedVectorType::get(IntTy, cast<FixedVectorType>(Op0->getType()));
21634 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
21635 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
21636
21637 SubVecTy = FixedVectorType::get(IntTy, LaneLen);
21638 }
21639
21640 // The base address of the store.
21641 Value *BaseAddr = SI->getPointerOperand();
21642
21643 if (NumStores > 1) {
21644 // If we're going to generate more than one store, reset the lane length
21645 // and sub-vector type to something legal.
21646 LaneLen /= NumStores;
21647 SubVecTy = FixedVectorType::get(SubVecTy->getElementType(), LaneLen);
21648
21649 // We will compute the pointer operand of each store from the original base
21650 // address using GEPs. Cast the base address to a pointer to the scalar
21651 // element type.
21652 BaseAddr = Builder.CreateBitCast(
21653 BaseAddr,
21654 SubVecTy->getElementType()->getPointerTo(SI->getPointerAddressSpace()));
21655 }
21656
21657 assert(isTypeLegal(EVT::getEVT(SubVecTy)) && "Illegal vstN vector type!");
21658
21659 auto Mask = SVI->getShuffleMask();
21660
21661 auto createStoreIntrinsic = [&](Value *BaseAddr,
21662 SmallVectorImpl<Value *> &Shuffles) {
21663 if (Subtarget->hasNEON()) {
21664 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21665 Intrinsic::arm_neon_vst3,
21666 Intrinsic::arm_neon_vst4};
21667 Type *Int8Ptr = Builder.getInt8PtrTy(SI->getPointerAddressSpace());
21668 Type *Tys[] = {Int8Ptr, SubVecTy};
21669
21670 Function *VstNFunc = Intrinsic::getDeclaration(
21671 SI->getModule(), StoreInts[Factor - 2], Tys);
21672
21673 SmallVector<Value *, 6> Ops;
21674 Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr));
21675 append_range(Ops, Shuffles);
21676 Ops.push_back(Builder.getInt32(SI->getAlign().value()));
21677 Builder.CreateCall(VstNFunc, Ops);
21678 } else {
21679 assert((Factor == 2 || Factor == 4) &&
21680 "expected interleave factor of 2 or 4 for MVE");
21681 Intrinsic::ID StoreInts =
21682 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21683 Type *EltPtrTy = SubVecTy->getElementType()->getPointerTo(
21684 SI->getPointerAddressSpace());
21685 Type *Tys[] = {EltPtrTy, SubVecTy};
21686 Function *VstNFunc =
21687 Intrinsic::getDeclaration(SI->getModule(), StoreInts, Tys);
21688
21689 SmallVector<Value *, 6> Ops;
21690 Ops.push_back(Builder.CreateBitCast(BaseAddr, EltPtrTy));
21691 append_range(Ops, Shuffles);
21692 for (unsigned F = 0; F < Factor; F++) {
21693 Ops.push_back(Builder.getInt32(F));
21694 Builder.CreateCall(VstNFunc, Ops);
21695 Ops.pop_back();
21696 }
21697 }
21698 };
21699
21700 for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21701 // If we generating more than one store, we compute the base address of
21702 // subsequent stores as an offset from the previous.
21703 if (StoreCount > 0)
21704 BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(),
21705 BaseAddr, LaneLen * Factor);
21706
21707 SmallVector<Value *, 4> Shuffles;
21708
21709 // Split the shufflevector operands into sub vectors for the new vstN call.
21710 for (unsigned i = 0; i < Factor; i++) {
21711 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21712 if (Mask[IdxI] >= 0) {
21713 Shuffles.push_back(Builder.CreateShuffleVector(
21714 Op0, Op1, createSequentialMask(Mask[IdxI], LaneLen, 0)));
21715 } else {
21716 unsigned StartMask = 0;
21717 for (unsigned j = 1; j < LaneLen; j++) {
21718 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21719 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21720 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21721 break;
21722 }
21723 }
21724 // Note: If all elements in a chunk are undefs, StartMask=0!
21725 // Note: Filling undef gaps with random elements is ok, since
21726 // those elements were being written anyway (with undefs).
21727 // In the case of all undefs we're defaulting to using elems from 0
21728 // Note: StartMask cannot be negative, it's checked in
21729 // isReInterleaveMask
21730 Shuffles.push_back(Builder.CreateShuffleVector(
21731 Op0, Op1, createSequentialMask(StartMask, LaneLen, 0)));
21732 }
21733 }
21734
21735 createStoreIntrinsic(BaseAddr, Shuffles);
21736 }
21737 return true;
21738 }
21739
21740 enum HABaseType {
21741 HA_UNKNOWN = 0,
21742 HA_FLOAT,
21743 HA_DOUBLE,
21744 HA_VECT64,
21745 HA_VECT128
21746 };
21747
isHomogeneousAggregate(Type * Ty,HABaseType & Base,uint64_t & Members)21748 static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
21749 uint64_t &Members) {
21750 if (auto *ST = dyn_cast<StructType>(Ty)) {
21751 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
21752 uint64_t SubMembers = 0;
21753 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers))
21754 return false;
21755 Members += SubMembers;
21756 }
21757 } else if (auto *AT = dyn_cast<ArrayType>(Ty)) {
21758 uint64_t SubMembers = 0;
21759 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers))
21760 return false;
21761 Members += SubMembers * AT->getNumElements();
21762 } else if (Ty->isFloatTy()) {
21763 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
21764 return false;
21765 Members = 1;
21766 Base = HA_FLOAT;
21767 } else if (Ty->isDoubleTy()) {
21768 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
21769 return false;
21770 Members = 1;
21771 Base = HA_DOUBLE;
21772 } else if (auto *VT = dyn_cast<VectorType>(Ty)) {
21773 Members = 1;
21774 switch (Base) {
21775 case HA_FLOAT:
21776 case HA_DOUBLE:
21777 return false;
21778 case HA_VECT64:
21779 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
21780 case HA_VECT128:
21781 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
21782 case HA_UNKNOWN:
21783 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
21784 case 64:
21785 Base = HA_VECT64;
21786 return true;
21787 case 128:
21788 Base = HA_VECT128;
21789 return true;
21790 default:
21791 return false;
21792 }
21793 }
21794 }
21795
21796 return (Members > 0 && Members <= 4);
21797 }
21798
21799 /// Return the correct alignment for the current calling convention.
getABIAlignmentForCallingConv(Type * ArgTy,const DataLayout & DL) const21800 Align ARMTargetLowering::getABIAlignmentForCallingConv(
21801 Type *ArgTy, const DataLayout &DL) const {
21802 const Align ABITypeAlign = DL.getABITypeAlign(ArgTy);
21803 if (!ArgTy->isVectorTy())
21804 return ABITypeAlign;
21805
21806 // Avoid over-aligning vector parameters. It would require realigning the
21807 // stack and waste space for no real benefit.
21808 return std::min(ABITypeAlign, DL.getStackAlignment());
21809 }
21810
21811 /// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
21812 /// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
21813 /// passing according to AAPCS rules.
functionArgumentNeedsConsecutiveRegisters(Type * Ty,CallingConv::ID CallConv,bool isVarArg,const DataLayout & DL) const21814 bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters(
21815 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
21816 const DataLayout &DL) const {
21817 if (getEffectiveCallingConv(CallConv, isVarArg) !=
21818 CallingConv::ARM_AAPCS_VFP)
21819 return false;
21820
21821 HABaseType Base = HA_UNKNOWN;
21822 uint64_t Members = 0;
21823 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
21824 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump());
21825
21826 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
21827 return IsHA || IsIntArray;
21828 }
21829
getExceptionPointerRegister(const Constant * PersonalityFn) const21830 Register ARMTargetLowering::getExceptionPointerRegister(
21831 const Constant *PersonalityFn) const {
21832 // Platforms which do not use SjLj EH may return values in these registers
21833 // via the personality function.
21834 return Subtarget->useSjLjEH() ? Register() : ARM::R0;
21835 }
21836
getExceptionSelectorRegister(const Constant * PersonalityFn) const21837 Register ARMTargetLowering::getExceptionSelectorRegister(
21838 const Constant *PersonalityFn) const {
21839 // Platforms which do not use SjLj EH may return values in these registers
21840 // via the personality function.
21841 return Subtarget->useSjLjEH() ? Register() : ARM::R1;
21842 }
21843
initializeSplitCSR(MachineBasicBlock * Entry) const21844 void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
21845 // Update IsSplitCSR in ARMFunctionInfo.
21846 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>();
21847 AFI->setIsSplitCSR(true);
21848 }
21849
insertCopiesSplitCSR(MachineBasicBlock * Entry,const SmallVectorImpl<MachineBasicBlock * > & Exits) const21850 void ARMTargetLowering::insertCopiesSplitCSR(
21851 MachineBasicBlock *Entry,
21852 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
21853 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
21854 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
21855 if (!IStart)
21856 return;
21857
21858 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21859 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
21860 MachineBasicBlock::iterator MBBI = Entry->begin();
21861 for (const MCPhysReg *I = IStart; *I; ++I) {
21862 const TargetRegisterClass *RC = nullptr;
21863 if (ARM::GPRRegClass.contains(*I))
21864 RC = &ARM::GPRRegClass;
21865 else if (ARM::DPRRegClass.contains(*I))
21866 RC = &ARM::DPRRegClass;
21867 else
21868 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
21869
21870 Register NewVR = MRI->createVirtualRegister(RC);
21871 // Create copy from CSR to a virtual register.
21872 // FIXME: this currently does not emit CFI pseudo-instructions, it works
21873 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
21874 // nounwind. If we want to generalize this later, we may need to emit
21875 // CFI pseudo-instructions.
21876 assert(Entry->getParent()->getFunction().hasFnAttribute(
21877 Attribute::NoUnwind) &&
21878 "Function should be nounwind in insertCopiesSplitCSR!");
21879 Entry->addLiveIn(*I);
21880 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
21881 .addReg(*I);
21882
21883 // Insert the copy-back instructions right before the terminator.
21884 for (auto *Exit : Exits)
21885 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
21886 TII->get(TargetOpcode::COPY), *I)
21887 .addReg(NewVR);
21888 }
21889 }
21890
finalizeLowering(MachineFunction & MF) const21891 void ARMTargetLowering::finalizeLowering(MachineFunction &MF) const {
21892 MF.getFrameInfo().computeMaxCallFrameSize(MF);
21893 TargetLoweringBase::finalizeLowering(MF);
21894 }
21895
isComplexDeinterleavingSupported() const21896 bool ARMTargetLowering::isComplexDeinterleavingSupported() const {
21897 return Subtarget->hasMVEIntegerOps();
21898 }
21899
isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation,Type * Ty) const21900 bool ARMTargetLowering::isComplexDeinterleavingOperationSupported(
21901 ComplexDeinterleavingOperation Operation, Type *Ty) const {
21902 auto *VTy = dyn_cast<FixedVectorType>(Ty);
21903 if (!VTy)
21904 return false;
21905
21906 auto *ScalarTy = VTy->getScalarType();
21907 unsigned NumElements = VTy->getNumElements();
21908
21909 unsigned VTyWidth = VTy->getScalarSizeInBits() * NumElements;
21910 if (VTyWidth < 128 || !llvm::isPowerOf2_32(VTyWidth))
21911 return false;
21912
21913 // Both VCADD and VCMUL/VCMLA support the same types, F16 and F32
21914 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
21915 return Subtarget->hasMVEFloatOps();
21916
21917 if (Operation != ComplexDeinterleavingOperation::CAdd)
21918 return false;
21919
21920 return Subtarget->hasMVEIntegerOps() &&
21921 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
21922 ScalarTy->isIntegerTy(32));
21923 }
21924
createComplexDeinterleavingIR(Instruction * I,ComplexDeinterleavingOperation OperationType,ComplexDeinterleavingRotation Rotation,Value * InputA,Value * InputB,Value * Accumulator) const21925 Value *ARMTargetLowering::createComplexDeinterleavingIR(
21926 Instruction *I, ComplexDeinterleavingOperation OperationType,
21927 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
21928 Value *Accumulator) const {
21929
21930 FixedVectorType *Ty = cast<FixedVectorType>(InputA->getType());
21931
21932 IRBuilder<> B(I);
21933
21934 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
21935
21936 assert(TyWidth >= 128 && "Width of vector type must be at least 128 bits");
21937
21938 if (TyWidth > 128) {
21939 int Stride = Ty->getNumElements() / 2;
21940 auto SplitSeq = llvm::seq<int>(0, Ty->getNumElements());
21941 auto SplitSeqVec = llvm::to_vector(SplitSeq);
21942 ArrayRef<int> LowerSplitMask(&SplitSeqVec[0], Stride);
21943 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
21944
21945 auto *LowerSplitA = B.CreateShuffleVector(InputA, LowerSplitMask);
21946 auto *LowerSplitB = B.CreateShuffleVector(InputB, LowerSplitMask);
21947 auto *UpperSplitA = B.CreateShuffleVector(InputA, UpperSplitMask);
21948 auto *UpperSplitB = B.CreateShuffleVector(InputB, UpperSplitMask);
21949 Value *LowerSplitAcc = nullptr;
21950 Value *UpperSplitAcc = nullptr;
21951
21952 if (Accumulator) {
21953 LowerSplitAcc = B.CreateShuffleVector(Accumulator, LowerSplitMask);
21954 UpperSplitAcc = B.CreateShuffleVector(Accumulator, UpperSplitMask);
21955 }
21956
21957 auto *LowerSplitInt = createComplexDeinterleavingIR(
21958 I, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
21959 auto *UpperSplitInt = createComplexDeinterleavingIR(
21960 I, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
21961
21962 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
21963 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
21964 }
21965
21966 auto *IntTy = Type::getInt32Ty(B.getContext());
21967
21968 ConstantInt *ConstRotation = nullptr;
21969 if (OperationType == ComplexDeinterleavingOperation::CMulPartial) {
21970 ConstRotation = ConstantInt::get(IntTy, (int)Rotation);
21971
21972 if (Accumulator)
21973 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
21974 {ConstRotation, Accumulator, InputB, InputA});
21975 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
21976 {ConstRotation, InputB, InputA});
21977 }
21978
21979 if (OperationType == ComplexDeinterleavingOperation::CAdd) {
21980 // 1 means the value is not halved.
21981 auto *ConstHalving = ConstantInt::get(IntTy, 1);
21982
21983 if (Rotation == ComplexDeinterleavingRotation::Rotation_90)
21984 ConstRotation = ConstantInt::get(IntTy, 0);
21985 else if (Rotation == ComplexDeinterleavingRotation::Rotation_270)
21986 ConstRotation = ConstantInt::get(IntTy, 1);
21987
21988 if (!ConstRotation)
21989 return nullptr; // Invalid rotation for arm_mve_vcaddq
21990
21991 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
21992 {ConstHalving, ConstRotation, InputA, InputB});
21993 }
21994
21995 return nullptr;
21996 }
21997