1 //===- AArch64LowerHomogeneousPrologEpilog.cpp ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that lowers homogeneous prolog/epilog instructions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64InstrInfo.h"
14 #include "AArch64Subtarget.h"
15 #include "MCTargetDesc/AArch64InstPrinter.h"
16 #include "Utils/AArch64BaseInfo.h"
17 #include "llvm/CodeGen/MachineBasicBlock.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineFunctionPass.h"
20 #include "llvm/CodeGen/MachineInstr.h"
21 #include "llvm/CodeGen/MachineInstrBuilder.h"
22 #include "llvm/CodeGen/MachineModuleInfo.h"
23 #include "llvm/CodeGen/MachineOperand.h"
24 #include "llvm/CodeGen/TargetSubtargetInfo.h"
25 #include "llvm/IR/DebugLoc.h"
26 #include "llvm/IR/IRBuilder.h"
27 #include "llvm/Pass.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <optional>
30 #include <sstream>
31 
32 using namespace llvm;
33 
34 #define AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME                           \
35   "AArch64 homogeneous prolog/epilog lowering pass"
36 
37 cl::opt<int> FrameHelperSizeThreshold(
38     "frame-helper-size-threshold", cl::init(2), cl::Hidden,
39     cl::desc("The minimum number of instructions that are outlined in a frame "
40              "helper (default = 2)"));
41 
42 namespace {
43 
44 class AArch64LowerHomogeneousPE {
45 public:
46   const AArch64InstrInfo *TII;
47 
AArch64LowerHomogeneousPE(Module * M,MachineModuleInfo * MMI)48   AArch64LowerHomogeneousPE(Module *M, MachineModuleInfo *MMI)
49       : M(M), MMI(MMI) {}
50 
51   bool run();
52   bool runOnMachineFunction(MachineFunction &Fn);
53 
54 private:
55   Module *M;
56   MachineModuleInfo *MMI;
57 
58   bool runOnMBB(MachineBasicBlock &MBB);
59   bool runOnMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
60                MachineBasicBlock::iterator &NextMBBI);
61 
62   /// Lower a HOM_Prolog pseudo instruction into a helper call
63   /// or a sequence of homogeneous stores.
64   /// When a a fp setup follows, it can be optimized.
65   bool lowerProlog(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
66                    MachineBasicBlock::iterator &NextMBBI);
67   /// Lower a HOM_Epilog pseudo instruction into a helper call
68   /// or a sequence of homogeneous loads.
69   /// When a return follow, it can be optimized.
70   bool lowerEpilog(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
71                    MachineBasicBlock::iterator &NextMBBI);
72 };
73 
74 class AArch64LowerHomogeneousPrologEpilog : public ModulePass {
75 public:
76   static char ID;
77 
AArch64LowerHomogeneousPrologEpilog()78   AArch64LowerHomogeneousPrologEpilog() : ModulePass(ID) {
79     initializeAArch64LowerHomogeneousPrologEpilogPass(
80         *PassRegistry::getPassRegistry());
81   }
getAnalysisUsage(AnalysisUsage & AU) const82   void getAnalysisUsage(AnalysisUsage &AU) const override {
83     AU.addRequired<MachineModuleInfoWrapperPass>();
84     AU.addPreserved<MachineModuleInfoWrapperPass>();
85     AU.setPreservesAll();
86     ModulePass::getAnalysisUsage(AU);
87   }
88   bool runOnModule(Module &M) override;
89 
getPassName() const90   StringRef getPassName() const override {
91     return AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME;
92   }
93 };
94 
95 } // end anonymous namespace
96 
97 char AArch64LowerHomogeneousPrologEpilog::ID = 0;
98 
99 INITIALIZE_PASS(AArch64LowerHomogeneousPrologEpilog,
100                 "aarch64-lower-homogeneous-prolog-epilog",
101                 AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME, false, false)
102 
runOnModule(Module & M)103 bool AArch64LowerHomogeneousPrologEpilog::runOnModule(Module &M) {
104   if (skipModule(M))
105     return false;
106 
107   MachineModuleInfo *MMI =
108       &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
109   return AArch64LowerHomogeneousPE(&M, MMI).run();
110 }
111 
run()112 bool AArch64LowerHomogeneousPE::run() {
113   bool Changed = false;
114   for (auto &F : *M) {
115     if (F.empty())
116       continue;
117 
118     MachineFunction *MF = MMI->getMachineFunction(F);
119     if (!MF)
120       continue;
121     Changed |= runOnMachineFunction(*MF);
122   }
123 
124   return Changed;
125 }
126 enum FrameHelperType { Prolog, PrologFrame, Epilog, EpilogTail };
127 
128 /// Return a frame helper name with the given CSRs and the helper type.
129 /// For instance, a prolog helper that saves x19 and x20 is named as
130 /// OUTLINED_FUNCTION_PROLOG_x19x20.
getFrameHelperName(SmallVectorImpl<unsigned> & Regs,FrameHelperType Type,unsigned FpOffset)131 static std::string getFrameHelperName(SmallVectorImpl<unsigned> &Regs,
132                                       FrameHelperType Type, unsigned FpOffset) {
133   std::ostringstream RegStream;
134   switch (Type) {
135   case FrameHelperType::Prolog:
136     RegStream << "OUTLINED_FUNCTION_PROLOG_";
137     break;
138   case FrameHelperType::PrologFrame:
139     RegStream << "OUTLINED_FUNCTION_PROLOG_FRAME" << FpOffset << "_";
140     break;
141   case FrameHelperType::Epilog:
142     RegStream << "OUTLINED_FUNCTION_EPILOG_";
143     break;
144   case FrameHelperType::EpilogTail:
145     RegStream << "OUTLINED_FUNCTION_EPILOG_TAIL_";
146     break;
147   }
148 
149   for (auto Reg : Regs)
150     RegStream << AArch64InstPrinter::getRegisterName(Reg);
151 
152   return RegStream.str();
153 }
154 
155 /// Create a Function for the unique frame helper with the given name.
156 /// Return a newly created MachineFunction with an empty MachineBasicBlock.
createFrameHelperMachineFunction(Module * M,MachineModuleInfo * MMI,StringRef Name)157 static MachineFunction &createFrameHelperMachineFunction(Module *M,
158                                                          MachineModuleInfo *MMI,
159                                                          StringRef Name) {
160   LLVMContext &C = M->getContext();
161   Function *F = M->getFunction(Name);
162   assert(F == nullptr && "Function has been created before");
163   F = Function::Create(FunctionType::get(Type::getVoidTy(C), false),
164                        Function::ExternalLinkage, Name, M);
165   assert(F && "Function was null!");
166 
167   // Use ODR linkage to avoid duplication.
168   F->setLinkage(GlobalValue::LinkOnceODRLinkage);
169   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
170 
171   // Set no-opt/minsize, so we don't insert padding between outlined
172   // functions.
173   F->addFnAttr(Attribute::OptimizeNone);
174   F->addFnAttr(Attribute::NoInline);
175   F->addFnAttr(Attribute::MinSize);
176   F->addFnAttr(Attribute::Naked);
177 
178   MachineFunction &MF = MMI->getOrCreateMachineFunction(*F);
179   // Remove unnecessary register liveness and set NoVRegs.
180   MF.getProperties().reset(MachineFunctionProperties::Property::TracksLiveness);
181   MF.getProperties().reset(MachineFunctionProperties::Property::IsSSA);
182   MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
183   MF.getRegInfo().freezeReservedRegs(MF);
184 
185   // Create entry block.
186   BasicBlock *EntryBB = BasicBlock::Create(C, "entry", F);
187   IRBuilder<> Builder(EntryBB);
188   Builder.CreateRetVoid();
189 
190   // Insert the new block into the function.
191   MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
192   MF.insert(MF.begin(), MBB);
193 
194   return MF;
195 }
196 
197 /// Emit a store-pair instruction for frame-setup.
emitStore(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator Pos,const TargetInstrInfo & TII,unsigned Reg1,unsigned Reg2,int Offset,bool IsPreDec)198 static void emitStore(MachineFunction &MF, MachineBasicBlock &MBB,
199                       MachineBasicBlock::iterator Pos,
200                       const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
201                       int Offset, bool IsPreDec) {
202   bool IsFloat = AArch64::FPR64RegClass.contains(Reg1);
203   assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
204   unsigned Opc;
205   if (IsPreDec)
206     Opc = IsFloat ? AArch64::STPDpre : AArch64::STPXpre;
207   else
208     Opc = IsFloat ? AArch64::STPDi : AArch64::STPXi;
209 
210   MachineInstrBuilder MIB = BuildMI(MBB, Pos, DebugLoc(), TII.get(Opc));
211   if (IsPreDec)
212     MIB.addDef(AArch64::SP);
213   MIB.addReg(Reg2)
214       .addReg(Reg1)
215       .addReg(AArch64::SP)
216       .addImm(Offset)
217       .setMIFlag(MachineInstr::FrameSetup);
218 }
219 
220 /// Emit a load-pair instruction for frame-destroy.
emitLoad(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator Pos,const TargetInstrInfo & TII,unsigned Reg1,unsigned Reg2,int Offset,bool IsPostDec)221 static void emitLoad(MachineFunction &MF, MachineBasicBlock &MBB,
222                      MachineBasicBlock::iterator Pos,
223                      const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
224                      int Offset, bool IsPostDec) {
225   bool IsFloat = AArch64::FPR64RegClass.contains(Reg1);
226   assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
227   unsigned Opc;
228   if (IsPostDec)
229     Opc = IsFloat ? AArch64::LDPDpost : AArch64::LDPXpost;
230   else
231     Opc = IsFloat ? AArch64::LDPDi : AArch64::LDPXi;
232 
233   MachineInstrBuilder MIB = BuildMI(MBB, Pos, DebugLoc(), TII.get(Opc));
234   if (IsPostDec)
235     MIB.addDef(AArch64::SP);
236   MIB.addReg(Reg2, getDefRegState(true))
237       .addReg(Reg1, getDefRegState(true))
238       .addReg(AArch64::SP)
239       .addImm(Offset)
240       .setMIFlag(MachineInstr::FrameDestroy);
241 }
242 
243 /// Return a unique function if a helper can be formed with the given Regs
244 /// and frame type.
245 /// 1) _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x21x22:
246 ///    stp x22, x21, [sp, #-32]!    ; x29/x30 has been stored at the caller
247 ///    stp x20, x19, [sp, #16]
248 ///    ret
249 ///
250 /// 2) _OUTLINED_FUNCTION_PROLOG_FRAME32_x30x29x19x20x21x22:
251 ///    stp x22, x21, [sp, #-32]!    ; x29/x30 has been stored at the caller
252 ///    stp x20, x19, [sp, #16]
253 ///    add fp, sp, #32
254 ///    ret
255 ///
256 /// 3) _OUTLINED_FUNCTION_EPILOG_x30x29x19x20x21x22:
257 ///    mov x16, x30
258 ///    ldp x29, x30, [sp, #32]
259 ///    ldp x20, x19, [sp, #16]
260 ///    ldp x22, x21, [sp], #48
261 ///    ret x16
262 ///
263 /// 4) _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x21x22:
264 ///    ldp x29, x30, [sp, #32]
265 ///    ldp x20, x19, [sp, #16]
266 ///    ldp x22, x21, [sp], #48
267 ///    ret
268 /// @param M module
269 /// @param MMI machine module info
270 /// @param Regs callee save regs that the helper will handle
271 /// @param Type frame helper type
272 /// @return a helper function
getOrCreateFrameHelper(Module * M,MachineModuleInfo * MMI,SmallVectorImpl<unsigned> & Regs,FrameHelperType Type,unsigned FpOffset=0)273 static Function *getOrCreateFrameHelper(Module *M, MachineModuleInfo *MMI,
274                                         SmallVectorImpl<unsigned> &Regs,
275                                         FrameHelperType Type,
276                                         unsigned FpOffset = 0) {
277   assert(Regs.size() >= 2);
278   auto Name = getFrameHelperName(Regs, Type, FpOffset);
279   auto *F = M->getFunction(Name);
280   if (F)
281     return F;
282 
283   auto &MF = createFrameHelperMachineFunction(M, MMI, Name);
284   MachineBasicBlock &MBB = *MF.begin();
285   const TargetSubtargetInfo &STI = MF.getSubtarget();
286   const TargetInstrInfo &TII = *STI.getInstrInfo();
287 
288   int Size = (int)Regs.size();
289   switch (Type) {
290   case FrameHelperType::Prolog:
291   case FrameHelperType::PrologFrame: {
292     // Compute the remaining SP adjust beyond FP/LR.
293     auto LRIdx = std::distance(Regs.begin(), llvm::find(Regs, AArch64::LR));
294 
295     // If the register stored to the lowest address is not LR, we must subtract
296     // more from SP here.
297     if (LRIdx != Size - 2) {
298       assert(Regs[Size - 2] != AArch64::LR);
299       emitStore(MF, MBB, MBB.end(), TII, Regs[Size - 2], Regs[Size - 1],
300                 LRIdx - Size + 2, true);
301     }
302 
303     // Store CSRs in the reverse order.
304     for (int I = Size - 3; I >= 0; I -= 2) {
305       // FP/LR has been stored at call-site.
306       if (Regs[I - 1] == AArch64::LR)
307         continue;
308       emitStore(MF, MBB, MBB.end(), TII, Regs[I - 1], Regs[I], Size - I - 1,
309                 false);
310     }
311     if (Type == FrameHelperType::PrologFrame)
312       BuildMI(MBB, MBB.end(), DebugLoc(), TII.get(AArch64::ADDXri))
313           .addDef(AArch64::FP)
314           .addUse(AArch64::SP)
315           .addImm(FpOffset)
316           .addImm(0)
317           .setMIFlag(MachineInstr::FrameSetup);
318 
319     BuildMI(MBB, MBB.end(), DebugLoc(), TII.get(AArch64::RET))
320         .addReg(AArch64::LR);
321     break;
322   }
323   case FrameHelperType::Epilog:
324   case FrameHelperType::EpilogTail:
325     if (Type == FrameHelperType::Epilog)
326       // Stash LR to X16
327       BuildMI(MBB, MBB.end(), DebugLoc(), TII.get(AArch64::ORRXrs))
328           .addDef(AArch64::X16)
329           .addReg(AArch64::XZR)
330           .addUse(AArch64::LR)
331           .addImm(0);
332 
333     for (int I = 0; I < Size - 2; I += 2)
334       emitLoad(MF, MBB, MBB.end(), TII, Regs[I], Regs[I + 1], Size - I - 2,
335                false);
336     // Restore the last CSR with post-increment of SP.
337     emitLoad(MF, MBB, MBB.end(), TII, Regs[Size - 2], Regs[Size - 1], Size,
338              true);
339 
340     BuildMI(MBB, MBB.end(), DebugLoc(), TII.get(AArch64::RET))
341         .addReg(Type == FrameHelperType::Epilog ? AArch64::X16 : AArch64::LR);
342     break;
343   }
344 
345   return M->getFunction(Name);
346 }
347 
348 /// This function checks if a frame helper should be used for
349 /// HOM_Prolog/HOM_Epilog pseudo instruction expansion.
350 /// @param MBB machine basic block
351 /// @param NextMBBI  next instruction following HOM_Prolog/HOM_Epilog
352 /// @param Regs callee save registers that are saved or restored.
353 /// @param Type frame helper type
354 /// @return True if a use of helper is qualified.
shouldUseFrameHelper(MachineBasicBlock & MBB,MachineBasicBlock::iterator & NextMBBI,SmallVectorImpl<unsigned> & Regs,FrameHelperType Type)355 static bool shouldUseFrameHelper(MachineBasicBlock &MBB,
356                                  MachineBasicBlock::iterator &NextMBBI,
357                                  SmallVectorImpl<unsigned> &Regs,
358                                  FrameHelperType Type) {
359   const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
360   auto RegCount = Regs.size();
361   assert(RegCount > 0 && (RegCount % 2 == 0));
362   // # of instructions that will be outlined.
363   int InstCount = RegCount / 2;
364 
365   // Do not use a helper call when not saving LR.
366   if (!llvm::is_contained(Regs, AArch64::LR))
367     return false;
368 
369   switch (Type) {
370   case FrameHelperType::Prolog:
371     // Prolog helper cannot save FP/LR.
372     InstCount--;
373     break;
374   case FrameHelperType::PrologFrame: {
375     // Effecitvely no change in InstCount since FpAdjusment is included.
376     break;
377   }
378   case FrameHelperType::Epilog:
379     // Bail-out if X16 is live across the epilog helper because it is used in
380     // the helper to handle X30.
381     for (auto NextMI = NextMBBI; NextMI != MBB.end(); NextMI++) {
382       if (NextMI->readsRegister(AArch64::W16, TRI))
383         return false;
384     }
385     // Epilog may not be in the last block. Check the liveness in successors.
386     for (const MachineBasicBlock *SuccMBB : MBB.successors()) {
387       if (SuccMBB->isLiveIn(AArch64::W16) || SuccMBB->isLiveIn(AArch64::X16))
388         return false;
389     }
390     // No change in InstCount for the regular epilog case.
391     break;
392   case FrameHelperType::EpilogTail: {
393     // EpilogTail helper includes the caller's return.
394     if (NextMBBI == MBB.end())
395       return false;
396     if (NextMBBI->getOpcode() != AArch64::RET_ReallyLR)
397       return false;
398     InstCount++;
399     break;
400   }
401   }
402 
403   return InstCount >= FrameHelperSizeThreshold;
404 }
405 
406 /// Lower a HOM_Epilog pseudo instruction into a helper call while
407 /// creating the helper on demand. Or emit a sequence of loads in place when not
408 /// using a helper call.
409 ///
410 /// 1. With a helper including ret
411 ///    HOM_Epilog x30, x29, x19, x20, x21, x22              ; MBBI
412 ///    ret                                                  ; NextMBBI
413 ///    =>
414 ///    b _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x21x22
415 ///    ...                                                  ; NextMBBI
416 ///
417 /// 2. With a helper
418 ///    HOM_Epilog x30, x29, x19, x20, x21, x22
419 ///    =>
420 ///    bl _OUTLINED_FUNCTION_EPILOG_x30x29x19x20x21x22
421 ///
422 /// 3. Without a helper
423 ///    HOM_Epilog x30, x29, x19, x20, x21, x22
424 ///    =>
425 ///    ldp x29, x30, [sp, #32]
426 ///    ldp x20, x19, [sp, #16]
427 ///    ldp x22, x21, [sp], #48
lowerEpilog(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineBasicBlock::iterator & NextMBBI)428 bool AArch64LowerHomogeneousPE::lowerEpilog(
429     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
430     MachineBasicBlock::iterator &NextMBBI) {
431   auto &MF = *MBB.getParent();
432   MachineInstr &MI = *MBBI;
433 
434   DebugLoc DL = MI.getDebugLoc();
435   SmallVector<unsigned, 8> Regs;
436   for (auto &MO : MI.operands())
437     if (MO.isReg())
438       Regs.push_back(MO.getReg());
439   int Size = (int)Regs.size();
440   if (Size == 0)
441     return false;
442   // Registers are in pair.
443   assert(Size % 2 == 0);
444   assert(MI.getOpcode() == AArch64::HOM_Epilog);
445 
446   auto Return = NextMBBI;
447   if (shouldUseFrameHelper(MBB, NextMBBI, Regs, FrameHelperType::EpilogTail)) {
448     // When MBB ends with a return, emit a tail-call to the epilog helper
449     auto *EpilogTailHelper =
450         getOrCreateFrameHelper(M, MMI, Regs, FrameHelperType::EpilogTail);
451     BuildMI(MBB, MBBI, DL, TII->get(AArch64::TCRETURNdi))
452         .addGlobalAddress(EpilogTailHelper)
453         .addImm(0)
454         .setMIFlag(MachineInstr::FrameDestroy)
455         .copyImplicitOps(MI)
456         .copyImplicitOps(*Return);
457     NextMBBI = std::next(Return);
458     Return->removeFromParent();
459   } else if (shouldUseFrameHelper(MBB, NextMBBI, Regs,
460                                   FrameHelperType::Epilog)) {
461     // The default epilog helper case.
462     auto *EpilogHelper =
463         getOrCreateFrameHelper(M, MMI, Regs, FrameHelperType::Epilog);
464     BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
465         .addGlobalAddress(EpilogHelper)
466         .setMIFlag(MachineInstr::FrameDestroy)
467         .copyImplicitOps(MI);
468   } else {
469     // Fall back to no-helper.
470     for (int I = 0; I < Size - 2; I += 2)
471       emitLoad(MF, MBB, MBBI, *TII, Regs[I], Regs[I + 1], Size - I - 2, false);
472     // Restore the last CSR with post-increment of SP.
473     emitLoad(MF, MBB, MBBI, *TII, Regs[Size - 2], Regs[Size - 1], Size, true);
474   }
475 
476   MBBI->removeFromParent();
477   return true;
478 }
479 
480 /// Lower a HOM_Prolog pseudo instruction into a helper call while
481 /// creating the helper on demand. Or emit a sequence of stores in place when
482 /// not using a helper call.
483 ///
484 /// 1. With a helper including frame-setup
485 ///    HOM_Prolog x30, x29, x19, x20, x21, x22, 32
486 ///    =>
487 ///    stp x29, x30, [sp, #-16]!
488 ///    bl _OUTLINED_FUNCTION_PROLOG_FRAME32_x30x29x19x20x21x22
489 ///
490 /// 2. With a helper
491 ///    HOM_Prolog x30, x29, x19, x20, x21, x22
492 ///    =>
493 ///    stp x29, x30, [sp, #-16]!
494 ///    bl _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x21x22
495 ///
496 /// 3. Without a helper
497 ///    HOM_Prolog x30, x29, x19, x20, x21, x22
498 ///    =>
499 ///    stp	x22, x21, [sp, #-48]!
500 ///    stp	x20, x19, [sp, #16]
501 ///    stp	x29, x30, [sp, #32]
lowerProlog(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineBasicBlock::iterator & NextMBBI)502 bool AArch64LowerHomogeneousPE::lowerProlog(
503     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
504     MachineBasicBlock::iterator &NextMBBI) {
505   auto &MF = *MBB.getParent();
506   MachineInstr &MI = *MBBI;
507 
508   DebugLoc DL = MI.getDebugLoc();
509   SmallVector<unsigned, 8> Regs;
510   int LRIdx = 0;
511   std::optional<int> FpOffset;
512   for (auto &MO : MI.operands()) {
513     if (MO.isReg()) {
514       if (MO.getReg() == AArch64::LR)
515         LRIdx = Regs.size();
516       Regs.push_back(MO.getReg());
517     } else if (MO.isImm()) {
518       FpOffset = MO.getImm();
519     }
520   }
521   int Size = (int)Regs.size();
522   if (Size == 0)
523     return false;
524   // Allow compact unwind case only for oww.
525   assert(Size % 2 == 0);
526   assert(MI.getOpcode() == AArch64::HOM_Prolog);
527 
528   if (FpOffset &&
529       shouldUseFrameHelper(MBB, NextMBBI, Regs, FrameHelperType::PrologFrame)) {
530     // FP/LR is stored at the top of stack before the prolog helper call.
531     emitStore(MF, MBB, MBBI, *TII, AArch64::LR, AArch64::FP, -LRIdx - 2, true);
532     auto *PrologFrameHelper = getOrCreateFrameHelper(
533         M, MMI, Regs, FrameHelperType::PrologFrame, *FpOffset);
534     BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
535         .addGlobalAddress(PrologFrameHelper)
536         .setMIFlag(MachineInstr::FrameSetup)
537         .copyImplicitOps(MI)
538         .addReg(AArch64::FP, RegState::Implicit | RegState::Define)
539         .addReg(AArch64::SP, RegState::Implicit);
540   } else if (!FpOffset && shouldUseFrameHelper(MBB, NextMBBI, Regs,
541                                                FrameHelperType::Prolog)) {
542     // FP/LR is stored at the top of stack before the prolog helper call.
543     emitStore(MF, MBB, MBBI, *TII, AArch64::LR, AArch64::FP, -LRIdx - 2, true);
544     auto *PrologHelper =
545         getOrCreateFrameHelper(M, MMI, Regs, FrameHelperType::Prolog);
546     BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
547         .addGlobalAddress(PrologHelper)
548         .setMIFlag(MachineInstr::FrameSetup)
549         .copyImplicitOps(MI);
550   } else {
551     // Fall back to no-helper.
552     emitStore(MF, MBB, MBBI, *TII, Regs[Size - 2], Regs[Size - 1], -Size, true);
553     for (int I = Size - 3; I >= 0; I -= 2)
554       emitStore(MF, MBB, MBBI, *TII, Regs[I - 1], Regs[I], Size - I - 1, false);
555     if (FpOffset) {
556       BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri))
557           .addDef(AArch64::FP)
558           .addUse(AArch64::SP)
559           .addImm(*FpOffset)
560           .addImm(0)
561           .setMIFlag(MachineInstr::FrameSetup);
562     }
563   }
564 
565   MBBI->removeFromParent();
566   return true;
567 }
568 
569 /// Process each machine instruction
570 /// @param MBB machine basic block
571 /// @param MBBI current instruction iterator
572 /// @param NextMBBI next instruction iterator which can be updated
573 /// @return True when IR is changed.
runOnMI(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineBasicBlock::iterator & NextMBBI)574 bool AArch64LowerHomogeneousPE::runOnMI(MachineBasicBlock &MBB,
575                                         MachineBasicBlock::iterator MBBI,
576                                         MachineBasicBlock::iterator &NextMBBI) {
577   MachineInstr &MI = *MBBI;
578   unsigned Opcode = MI.getOpcode();
579   switch (Opcode) {
580   default:
581     break;
582   case AArch64::HOM_Prolog:
583     return lowerProlog(MBB, MBBI, NextMBBI);
584   case AArch64::HOM_Epilog:
585     return lowerEpilog(MBB, MBBI, NextMBBI);
586   }
587   return false;
588 }
589 
runOnMBB(MachineBasicBlock & MBB)590 bool AArch64LowerHomogeneousPE::runOnMBB(MachineBasicBlock &MBB) {
591   bool Modified = false;
592 
593   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
594   while (MBBI != E) {
595     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
596     Modified |= runOnMI(MBB, MBBI, NMBBI);
597     MBBI = NMBBI;
598   }
599 
600   return Modified;
601 }
602 
runOnMachineFunction(MachineFunction & MF)603 bool AArch64LowerHomogeneousPE::runOnMachineFunction(MachineFunction &MF) {
604   TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
605 
606   bool Modified = false;
607   for (auto &MBB : MF)
608     Modified |= runOnMBB(MBB);
609   return Modified;
610 }
611 
createAArch64LowerHomogeneousPrologEpilogPass()612 ModulePass *llvm::createAArch64LowerHomogeneousPrologEpilogPass() {
613   return new AArch64LowerHomogeneousPrologEpilog();
614 }
615