1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of HWAddressSanitizer, an address basic correctness
11 /// checker based on tagged addressing.
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15 #include "llvm/ADT/MapVector.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/Analysis/GlobalsModRef.h"
22 #include "llvm/Analysis/PostDominators.h"
23 #include "llvm/Analysis/StackSafetyAnalysis.h"
24 #include "llvm/Analysis/ValueTracking.h"
25 #include "llvm/BinaryFormat/Dwarf.h"
26 #include "llvm/BinaryFormat/ELF.h"
27 #include "llvm/IR/Attributes.h"
28 #include "llvm/IR/BasicBlock.h"
29 #include "llvm/IR/Constant.h"
30 #include "llvm/IR/Constants.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/DebugInfoMetadata.h"
33 #include "llvm/IR/DerivedTypes.h"
34 #include "llvm/IR/Dominators.h"
35 #include "llvm/IR/Function.h"
36 #include "llvm/IR/IRBuilder.h"
37 #include "llvm/IR/InlineAsm.h"
38 #include "llvm/IR/InstIterator.h"
39 #include "llvm/IR/Instruction.h"
40 #include "llvm/IR/Instructions.h"
41 #include "llvm/IR/IntrinsicInst.h"
42 #include "llvm/IR/Intrinsics.h"
43 #include "llvm/IR/LLVMContext.h"
44 #include "llvm/IR/MDBuilder.h"
45 #include "llvm/IR/Module.h"
46 #include "llvm/IR/NoFolder.h"
47 #include "llvm/IR/Type.h"
48 #include "llvm/IR/Value.h"
49 #include "llvm/Support/Casting.h"
50 #include "llvm/Support/CommandLine.h"
51 #include "llvm/Support/Debug.h"
52 #include "llvm/Support/raw_ostream.h"
53 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
54 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
55 #include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
56 #include "llvm/Transforms/Utils/ModuleUtils.h"
57 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
58 #include <optional>
59 
60 using namespace llvm;
61 
62 #define DEBUG_TYPE "hwasan"
63 
64 const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
65 const char kHwasanNoteName[] = "hwasan.note";
66 const char kHwasanInitName[] = "__hwasan_init";
67 const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
68 
69 const char kHwasanShadowMemoryDynamicAddress[] =
70     "__hwasan_shadow_memory_dynamic_address";
71 
72 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
73 static const size_t kNumberOfAccessSizes = 5;
74 
75 static const size_t kDefaultShadowScale = 4;
76 static const uint64_t kDynamicShadowSentinel =
77     std::numeric_limits<uint64_t>::max();
78 
79 static const unsigned kShadowBaseAlignment = 32;
80 
81 static cl::opt<std::string>
82     ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix",
83                                  cl::desc("Prefix for memory access callbacks"),
84                                  cl::Hidden, cl::init("__hwasan_"));
85 
86 static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
87     "hwasan-kernel-mem-intrinsic-prefix",
88     cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
89     cl::init(false));
90 
91 static cl::opt<bool> ClInstrumentWithCalls(
92     "hwasan-instrument-with-calls",
93     cl::desc("instrument reads and writes with callbacks"), cl::Hidden,
94     cl::init(false));
95 
96 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
97                                        cl::desc("instrument read instructions"),
98                                        cl::Hidden, cl::init(true));
99 
100 static cl::opt<bool>
101     ClInstrumentWrites("hwasan-instrument-writes",
102                        cl::desc("instrument write instructions"), cl::Hidden,
103                        cl::init(true));
104 
105 static cl::opt<bool> ClInstrumentAtomics(
106     "hwasan-instrument-atomics",
107     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
108     cl::init(true));
109 
110 static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
111                                        cl::desc("instrument byval arguments"),
112                                        cl::Hidden, cl::init(true));
113 
114 static cl::opt<bool>
115     ClRecover("hwasan-recover",
116               cl::desc("Enable recovery mode (continue-after-error)."),
117               cl::Hidden, cl::init(false));
118 
119 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
120                                        cl::desc("instrument stack (allocas)"),
121                                        cl::Hidden, cl::init(true));
122 
123 static cl::opt<bool>
124     ClUseStackSafety("hwasan-use-stack-safety", cl::Hidden, cl::init(true),
125                      cl::Hidden, cl::desc("Use Stack Safety analysis results"),
126                      cl::Optional);
127 
128 static cl::opt<size_t> ClMaxLifetimes(
129     "hwasan-max-lifetimes-for-alloca", cl::Hidden, cl::init(3),
130     cl::ReallyHidden,
131     cl::desc("How many lifetime ends to handle for a single alloca."),
132     cl::Optional);
133 
134 static cl::opt<bool>
135     ClUseAfterScope("hwasan-use-after-scope",
136                     cl::desc("detect use after scope within function"),
137                     cl::Hidden, cl::init(false));
138 
139 static cl::opt<bool> ClUARRetagToZero(
140     "hwasan-uar-retag-to-zero",
141     cl::desc("Clear alloca tags before returning from the function to allow "
142              "non-instrumented and instrumented function calls mix. When set "
143              "to false, allocas are retagged before returning from the "
144              "function to detect use after return."),
145     cl::Hidden, cl::init(true));
146 
147 static cl::opt<bool> ClGenerateTagsWithCalls(
148     "hwasan-generate-tags-with-calls",
149     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
150     cl::init(false));
151 
152 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
153                                cl::Hidden, cl::init(false));
154 
155 static cl::opt<int> ClMatchAllTag(
156     "hwasan-match-all-tag",
157     cl::desc("don't report bad accesses via pointers with this tag"),
158     cl::Hidden, cl::init(-1));
159 
160 static cl::opt<bool>
161     ClEnableKhwasan("hwasan-kernel",
162                     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
163                     cl::Hidden, cl::init(false));
164 
165 // These flags allow to change the shadow mapping and control how shadow memory
166 // is accessed. The shadow mapping looks like:
167 //    Shadow = (Mem >> scale) + offset
168 
169 static cl::opt<uint64_t>
170     ClMappingOffset("hwasan-mapping-offset",
171                     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
172                     cl::Hidden, cl::init(0));
173 
174 static cl::opt<bool>
175     ClWithIfunc("hwasan-with-ifunc",
176                 cl::desc("Access dynamic shadow through an ifunc global on "
177                          "platforms that support this"),
178                 cl::Hidden, cl::init(false));
179 
180 static cl::opt<bool> ClWithTls(
181     "hwasan-with-tls",
182     cl::desc("Access dynamic shadow through an thread-local pointer on "
183              "platforms that support this"),
184     cl::Hidden, cl::init(true));
185 
186 // Mode for selecting how to insert frame record info into the stack ring
187 // buffer.
188 enum RecordStackHistoryMode {
189   // Do not record frame record info.
190   none,
191 
192   // Insert instructions into the prologue for storing into the stack ring
193   // buffer directly.
194   instr,
195 
196   // Add a call to __hwasan_add_frame_record in the runtime.
197   libcall,
198 };
199 
200 static cl::opt<RecordStackHistoryMode> ClRecordStackHistory(
201     "hwasan-record-stack-history",
202     cl::desc("Record stack frames with tagged allocations in a thread-local "
203              "ring buffer"),
204     cl::values(clEnumVal(none, "Do not record stack ring history"),
205                clEnumVal(instr, "Insert instructions into the prologue for "
206                                 "storing into the stack ring buffer directly"),
207                clEnumVal(libcall, "Add a call to __hwasan_add_frame_record for "
208                                   "storing into the stack ring buffer")),
209     cl::Hidden, cl::init(instr));
210 
211 static cl::opt<bool>
212     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
213                               cl::desc("instrument memory intrinsics"),
214                               cl::Hidden, cl::init(true));
215 
216 static cl::opt<bool>
217     ClInstrumentLandingPads("hwasan-instrument-landing-pads",
218                             cl::desc("instrument landing pads"), cl::Hidden,
219                             cl::init(false));
220 
221 static cl::opt<bool> ClUseShortGranules(
222     "hwasan-use-short-granules",
223     cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
224     cl::init(false));
225 
226 static cl::opt<bool> ClInstrumentPersonalityFunctions(
227     "hwasan-instrument-personality-functions",
228     cl::desc("instrument personality functions"), cl::Hidden);
229 
230 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
231                                        cl::desc("inline all checks"),
232                                        cl::Hidden, cl::init(false));
233 
234 // Enabled from clang by "-fsanitize-hwaddress-experimental-aliasing".
235 static cl::opt<bool> ClUsePageAliases("hwasan-experimental-use-page-aliases",
236                                       cl::desc("Use page aliasing in HWASan"),
237                                       cl::Hidden, cl::init(false));
238 
239 namespace {
240 
shouldUsePageAliases(const Triple & TargetTriple)241 bool shouldUsePageAliases(const Triple &TargetTriple) {
242   return ClUsePageAliases && TargetTriple.getArch() == Triple::x86_64;
243 }
244 
shouldInstrumentStack(const Triple & TargetTriple)245 bool shouldInstrumentStack(const Triple &TargetTriple) {
246   return !shouldUsePageAliases(TargetTriple) && ClInstrumentStack;
247 }
248 
shouldInstrumentWithCalls(const Triple & TargetTriple)249 bool shouldInstrumentWithCalls(const Triple &TargetTriple) {
250   return ClInstrumentWithCalls || TargetTriple.getArch() == Triple::x86_64;
251 }
252 
mightUseStackSafetyAnalysis(bool DisableOptimization)253 bool mightUseStackSafetyAnalysis(bool DisableOptimization) {
254   return ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety
255                                               : !DisableOptimization;
256 }
257 
shouldUseStackSafetyAnalysis(const Triple & TargetTriple,bool DisableOptimization)258 bool shouldUseStackSafetyAnalysis(const Triple &TargetTriple,
259                                   bool DisableOptimization) {
260   return shouldInstrumentStack(TargetTriple) &&
261          mightUseStackSafetyAnalysis(DisableOptimization);
262 }
263 
shouldDetectUseAfterScope(const Triple & TargetTriple)264 bool shouldDetectUseAfterScope(const Triple &TargetTriple) {
265   return ClUseAfterScope && shouldInstrumentStack(TargetTriple);
266 }
267 
268 /// An instrumentation pass implementing detection of addressability bugs
269 /// using tagged pointers.
270 class HWAddressSanitizer {
271 public:
HWAddressSanitizer(Module & M,bool CompileKernel,bool Recover,const StackSafetyGlobalInfo * SSI)272   HWAddressSanitizer(Module &M, bool CompileKernel, bool Recover,
273                      const StackSafetyGlobalInfo *SSI)
274       : M(M), SSI(SSI) {
275     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
276     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0
277                               ? ClEnableKhwasan
278                               : CompileKernel;
279 
280     initializeModule();
281   }
282 
setSSI(const StackSafetyGlobalInfo * S)283   void setSSI(const StackSafetyGlobalInfo *S) { SSI = S; }
284 
285   bool sanitizeFunction(Function &F, FunctionAnalysisManager &FAM);
286   void initializeModule();
287   void createHwasanCtorComdat();
288 
289   void initializeCallbacks(Module &M);
290 
291   Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
292 
293   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
294   Value *getShadowNonTls(IRBuilder<> &IRB);
295 
296   void untagPointerOperand(Instruction *I, Value *Addr);
297   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
298 
299   int64_t getAccessInfo(bool IsWrite, unsigned AccessSizeIndex);
300   void instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
301                                   unsigned AccessSizeIndex,
302                                   Instruction *InsertBefore);
303   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
304                                  unsigned AccessSizeIndex,
305                                  Instruction *InsertBefore);
306   bool ignoreMemIntrinsic(MemIntrinsic *MI);
307   void instrumentMemIntrinsic(MemIntrinsic *MI);
308   bool instrumentMemAccess(InterestingMemoryOperand &O);
309   bool ignoreAccess(Instruction *Inst, Value *Ptr);
310   void getInterestingMemoryOperands(
311       Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
312 
313   void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
314   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
315   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
316   bool instrumentStack(memtag::StackInfo &Info, Value *StackTag,
317                        const DominatorTree &DT, const PostDominatorTree &PDT,
318                        const LoopInfo &LI);
319   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
320   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
321   Value *getNextTagWithCall(IRBuilder<> &IRB);
322   Value *getStackBaseTag(IRBuilder<> &IRB);
323   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
324                       unsigned AllocaNo);
325   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
326 
327   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
328   Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
329   unsigned retagMask(unsigned AllocaNo);
330 
331   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
332 
333   void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
334   void instrumentGlobals();
335 
336   Value *getPC(IRBuilder<> &IRB);
337   Value *getSP(IRBuilder<> &IRB);
338   Value *getFrameRecordInfo(IRBuilder<> &IRB);
339 
340   void instrumentPersonalityFunctions();
341 
342 private:
343   LLVMContext *C;
344   Module &M;
345   const StackSafetyGlobalInfo *SSI;
346   Triple TargetTriple;
347   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
348   FunctionCallee HWAsanHandleVfork;
349 
350   /// This struct defines the shadow mapping using the rule:
351   ///   shadow = (mem >> Scale) + Offset.
352   /// If InGlobal is true, then
353   ///   extern char __hwasan_shadow[];
354   ///   shadow = (mem >> Scale) + &__hwasan_shadow
355   /// If InTls is true, then
356   ///   extern char *__hwasan_tls;
357   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
358   ///
359   /// If WithFrameRecord is true, then __hwasan_tls will be used to access the
360   /// ring buffer for storing stack allocations on targets that support it.
361   struct ShadowMapping {
362     uint8_t Scale;
363     uint64_t Offset;
364     bool InGlobal;
365     bool InTls;
366     bool WithFrameRecord;
367 
368     void init(Triple &TargetTriple, bool InstrumentWithCalls);
getObjectAlignment__anon1617492f0111::HWAddressSanitizer::ShadowMapping369     Align getObjectAlignment() const { return Align(1ULL << Scale); }
370   };
371 
372   ShadowMapping Mapping;
373 
374   Type *VoidTy = Type::getVoidTy(M.getContext());
375   Type *IntptrTy;
376   Type *Int8PtrTy;
377   Type *Int8Ty;
378   Type *Int32Ty;
379   Type *Int64Ty = Type::getInt64Ty(M.getContext());
380 
381   bool CompileKernel;
382   bool Recover;
383   bool OutlinedChecks;
384   bool UseShortGranules;
385   bool InstrumentLandingPads;
386   bool InstrumentWithCalls;
387   bool InstrumentStack;
388   bool DetectUseAfterScope;
389   bool UsePageAliases;
390 
391   std::optional<uint8_t> MatchAllTag;
392 
393   unsigned PointerTagShift;
394   uint64_t TagMaskByte;
395 
396   Function *HwasanCtorFunction;
397 
398   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
399   FunctionCallee HwasanMemoryAccessCallbackSized[2];
400 
401   FunctionCallee HwasanTagMemoryFunc;
402   FunctionCallee HwasanGenerateTagFunc;
403   FunctionCallee HwasanRecordFrameRecordFunc;
404 
405   Constant *ShadowGlobal;
406 
407   Value *ShadowBase = nullptr;
408   Value *StackBaseTag = nullptr;
409   Value *CachedSP = nullptr;
410   GlobalValue *ThreadPtrGlobal = nullptr;
411 };
412 
413 } // end anonymous namespace
414 
run(Module & M,ModuleAnalysisManager & MAM)415 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
416                                               ModuleAnalysisManager &MAM) {
417   const StackSafetyGlobalInfo *SSI = nullptr;
418   auto TargetTriple = llvm::Triple(M.getTargetTriple());
419   if (shouldUseStackSafetyAnalysis(TargetTriple, Options.DisableOptimization))
420     SSI = &MAM.getResult<StackSafetyGlobalAnalysis>(M);
421 
422   HWAddressSanitizer HWASan(M, Options.CompileKernel, Options.Recover, SSI);
423   bool Modified = false;
424   auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
425   for (Function &F : M)
426     Modified |= HWASan.sanitizeFunction(F, FAM);
427   if (!Modified)
428     return PreservedAnalyses::all();
429 
430   PreservedAnalyses PA = PreservedAnalyses::none();
431   // GlobalsAA is considered stateless and does not get invalidated unless
432   // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
433   // make changes that require GlobalsAA to be invalidated.
434   PA.abandon<GlobalsAA>();
435   return PA;
436 }
printPipeline(raw_ostream & OS,function_ref<StringRef (StringRef)> MapClassName2PassName)437 void HWAddressSanitizerPass::printPipeline(
438     raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
439   static_cast<PassInfoMixin<HWAddressSanitizerPass> *>(this)->printPipeline(
440       OS, MapClassName2PassName);
441   OS << "<";
442   if (Options.CompileKernel)
443     OS << "kernel;";
444   if (Options.Recover)
445     OS << "recover";
446   OS << ">";
447 }
448 
createHwasanCtorComdat()449 void HWAddressSanitizer::createHwasanCtorComdat() {
450   std::tie(HwasanCtorFunction, std::ignore) =
451       getOrCreateSanitizerCtorAndInitFunctions(
452           M, kHwasanModuleCtorName, kHwasanInitName,
453           /*InitArgTypes=*/{},
454           /*InitArgs=*/{},
455           // This callback is invoked when the functions are created the first
456           // time. Hook them into the global ctors list in that case:
457           [&](Function *Ctor, FunctionCallee) {
458             Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
459             Ctor->setComdat(CtorComdat);
460             appendToGlobalCtors(M, Ctor, 0, Ctor);
461           });
462 
463   // Create a note that contains pointers to the list of global
464   // descriptors. Adding a note to the output file will cause the linker to
465   // create a PT_NOTE program header pointing to the note that we can use to
466   // find the descriptor list starting from the program headers. A function
467   // provided by the runtime initializes the shadow memory for the globals by
468   // accessing the descriptor list via the note. The dynamic loader needs to
469   // call this function whenever a library is loaded.
470   //
471   // The reason why we use a note for this instead of a more conventional
472   // approach of having a global constructor pass a descriptor list pointer to
473   // the runtime is because of an order of initialization problem. With
474   // constructors we can encounter the following problematic scenario:
475   //
476   // 1) library A depends on library B and also interposes one of B's symbols
477   // 2) B's constructors are called before A's (as required for correctness)
478   // 3) during construction, B accesses one of its "own" globals (actually
479   //    interposed by A) and triggers a HWASAN failure due to the initialization
480   //    for A not having happened yet
481   //
482   // Even without interposition it is possible to run into similar situations in
483   // cases where two libraries mutually depend on each other.
484   //
485   // We only need one note per binary, so put everything for the note in a
486   // comdat. This needs to be a comdat with an .init_array section to prevent
487   // newer versions of lld from discarding the note.
488   //
489   // Create the note even if we aren't instrumenting globals. This ensures that
490   // binaries linked from object files with both instrumented and
491   // non-instrumented globals will end up with a note, even if a comdat from an
492   // object file with non-instrumented globals is selected. The note is harmless
493   // if the runtime doesn't support it, since it will just be ignored.
494   Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
495 
496   Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
497   auto *Start =
498       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
499                          nullptr, "__start_hwasan_globals");
500   Start->setVisibility(GlobalValue::HiddenVisibility);
501   auto *Stop =
502       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
503                          nullptr, "__stop_hwasan_globals");
504   Stop->setVisibility(GlobalValue::HiddenVisibility);
505 
506   // Null-terminated so actually 8 bytes, which are required in order to align
507   // the note properly.
508   auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
509 
510   auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
511                                  Int32Ty, Int32Ty);
512   auto *Note =
513       new GlobalVariable(M, NoteTy, /*isConstant=*/true,
514                          GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
515   Note->setSection(".note.hwasan.globals");
516   Note->setComdat(NoteComdat);
517   Note->setAlignment(Align(4));
518 
519   // The pointers in the note need to be relative so that the note ends up being
520   // placed in rodata, which is the standard location for notes.
521   auto CreateRelPtr = [&](Constant *Ptr) {
522     return ConstantExpr::getTrunc(
523         ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
524                              ConstantExpr::getPtrToInt(Note, Int64Ty)),
525         Int32Ty);
526   };
527   Note->setInitializer(ConstantStruct::getAnon(
528       {ConstantInt::get(Int32Ty, 8),                           // n_namesz
529        ConstantInt::get(Int32Ty, 8),                           // n_descsz
530        ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
531        Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
532   appendToCompilerUsed(M, Note);
533 
534   // Create a zero-length global in hwasan_globals so that the linker will
535   // always create start and stop symbols.
536   auto *Dummy = new GlobalVariable(
537       M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
538       Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
539   Dummy->setSection("hwasan_globals");
540   Dummy->setComdat(NoteComdat);
541   Dummy->setMetadata(LLVMContext::MD_associated,
542                      MDNode::get(*C, ValueAsMetadata::get(Note)));
543   appendToCompilerUsed(M, Dummy);
544 }
545 
546 /// Module-level initialization.
547 ///
548 /// inserts a call to __hwasan_init to the module's constructor list.
initializeModule()549 void HWAddressSanitizer::initializeModule() {
550   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
551   auto &DL = M.getDataLayout();
552 
553   TargetTriple = Triple(M.getTargetTriple());
554 
555   // x86_64 currently has two modes:
556   // - Intel LAM (default)
557   // - pointer aliasing (heap only)
558   bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
559   UsePageAliases = shouldUsePageAliases(TargetTriple);
560   InstrumentWithCalls = shouldInstrumentWithCalls(TargetTriple);
561   InstrumentStack = shouldInstrumentStack(TargetTriple);
562   DetectUseAfterScope = shouldDetectUseAfterScope(TargetTriple);
563   PointerTagShift = IsX86_64 ? 57 : 56;
564   TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
565 
566   Mapping.init(TargetTriple, InstrumentWithCalls);
567 
568   C = &(M.getContext());
569   IRBuilder<> IRB(*C);
570   IntptrTy = IRB.getIntPtrTy(DL);
571   Int8PtrTy = IRB.getInt8PtrTy();
572   Int8Ty = IRB.getInt8Ty();
573   Int32Ty = IRB.getInt32Ty();
574 
575   HwasanCtorFunction = nullptr;
576 
577   // Older versions of Android do not have the required runtime support for
578   // short granules, global or personality function instrumentation. On other
579   // platforms we currently require using the latest version of the runtime.
580   bool NewRuntime =
581       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
582 
583   UseShortGranules =
584       ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
585   OutlinedChecks =
586       (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
587       TargetTriple.isOSBinFormatELF() &&
588       (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
589 
590   if (ClMatchAllTag.getNumOccurrences()) {
591     if (ClMatchAllTag != -1) {
592       MatchAllTag = ClMatchAllTag & 0xFF;
593     }
594   } else if (CompileKernel) {
595     MatchAllTag = 0xFF;
596   }
597 
598   // If we don't have personality function support, fall back to landing pads.
599   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
600                               ? ClInstrumentLandingPads
601                               : !NewRuntime;
602 
603   if (!CompileKernel) {
604     createHwasanCtorComdat();
605     bool InstrumentGlobals =
606         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
607 
608     if (InstrumentGlobals && !UsePageAliases)
609       instrumentGlobals();
610 
611     bool InstrumentPersonalityFunctions =
612         ClInstrumentPersonalityFunctions.getNumOccurrences()
613             ? ClInstrumentPersonalityFunctions
614             : NewRuntime;
615     if (InstrumentPersonalityFunctions)
616       instrumentPersonalityFunctions();
617   }
618 
619   if (!TargetTriple.isAndroid()) {
620     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
621       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
622                                     GlobalValue::ExternalLinkage, nullptr,
623                                     "__hwasan_tls", nullptr,
624                                     GlobalVariable::InitialExecTLSModel);
625       appendToCompilerUsed(M, GV);
626       return GV;
627     });
628     ThreadPtrGlobal = cast<GlobalVariable>(C);
629   }
630 }
631 
initializeCallbacks(Module & M)632 void HWAddressSanitizer::initializeCallbacks(Module &M) {
633   IRBuilder<> IRB(*C);
634   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
635     const std::string TypeStr = AccessIsWrite ? "store" : "load";
636     const std::string EndingStr = Recover ? "_noabort" : "";
637 
638     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
639         ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
640         FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
641 
642     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
643          AccessSizeIndex++) {
644       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
645           M.getOrInsertFunction(
646               ClMemoryAccessCallbackPrefix + TypeStr +
647                   itostr(1ULL << AccessSizeIndex) + EndingStr,
648               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
649     }
650   }
651 
652   HwasanTagMemoryFunc = M.getOrInsertFunction(
653       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
654   HwasanGenerateTagFunc =
655       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
656 
657   HwasanRecordFrameRecordFunc = M.getOrInsertFunction(
658       "__hwasan_add_frame_record", IRB.getVoidTy(), Int64Ty);
659 
660   ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
661                                      ArrayType::get(IRB.getInt8Ty(), 0));
662 
663   const std::string MemIntrinCallbackPrefix =
664       (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
665           ? std::string("")
666           : ClMemoryAccessCallbackPrefix;
667   HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
668                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
669                                         IRB.getInt8PtrTy(), IntptrTy);
670   HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
671                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
672                                        IRB.getInt8PtrTy(), IntptrTy);
673   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
674                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
675                                        IRB.getInt32Ty(), IntptrTy);
676 
677   HWAsanHandleVfork =
678       M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
679 }
680 
getOpaqueNoopCast(IRBuilder<> & IRB,Value * Val)681 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
682   // An empty inline asm with input reg == output reg.
683   // An opaque no-op cast, basically.
684   // This prevents code bloat as a result of rematerializing trivial definitions
685   // such as constants or global addresses at every load and store.
686   InlineAsm *Asm =
687       InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
688                      StringRef(""), StringRef("=r,0"),
689                      /*hasSideEffects=*/false);
690   return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
691 }
692 
getDynamicShadowIfunc(IRBuilder<> & IRB)693 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
694   return getOpaqueNoopCast(IRB, ShadowGlobal);
695 }
696 
getShadowNonTls(IRBuilder<> & IRB)697 Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
698   if (Mapping.Offset != kDynamicShadowSentinel)
699     return getOpaqueNoopCast(
700         IRB, ConstantExpr::getIntToPtr(
701                  ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
702 
703   if (Mapping.InGlobal)
704     return getDynamicShadowIfunc(IRB);
705 
706   Value *GlobalDynamicAddress =
707       IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
708           kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
709   return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
710 }
711 
ignoreAccess(Instruction * Inst,Value * Ptr)712 bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
713   // Do not instrument accesses from different address spaces; we cannot deal
714   // with them.
715   Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
716   if (PtrTy->getPointerAddressSpace() != 0)
717     return true;
718 
719   // Ignore swifterror addresses.
720   // swifterror memory addresses are mem2reg promoted by instruction
721   // selection. As such they cannot have regular uses like an instrumentation
722   // function and it makes no sense to track them as memory.
723   if (Ptr->isSwiftError())
724     return true;
725 
726   if (findAllocaForValue(Ptr)) {
727     if (!InstrumentStack)
728       return true;
729     if (SSI && SSI->stackAccessIsSafe(*Inst))
730       return true;
731   }
732   return false;
733 }
734 
getInterestingMemoryOperands(Instruction * I,SmallVectorImpl<InterestingMemoryOperand> & Interesting)735 void HWAddressSanitizer::getInterestingMemoryOperands(
736     Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
737   // Skip memory accesses inserted by another instrumentation.
738   if (I->hasMetadata(LLVMContext::MD_nosanitize))
739     return;
740 
741   // Do not instrument the load fetching the dynamic shadow address.
742   if (ShadowBase == I)
743     return;
744 
745   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
746     if (!ClInstrumentReads || ignoreAccess(I, LI->getPointerOperand()))
747       return;
748     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
749                              LI->getType(), LI->getAlign());
750   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
751     if (!ClInstrumentWrites || ignoreAccess(I, SI->getPointerOperand()))
752       return;
753     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
754                              SI->getValueOperand()->getType(), SI->getAlign());
755   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
756     if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
757       return;
758     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
759                              RMW->getValOperand()->getType(), std::nullopt);
760   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
761     if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
762       return;
763     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
764                              XCHG->getCompareOperand()->getType(),
765                              std::nullopt);
766   } else if (auto *CI = dyn_cast<CallInst>(I)) {
767     for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
768       if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
769           ignoreAccess(I, CI->getArgOperand(ArgNo)))
770         continue;
771       Type *Ty = CI->getParamByValType(ArgNo);
772       Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
773     }
774   }
775 }
776 
getPointerOperandIndex(Instruction * I)777 static unsigned getPointerOperandIndex(Instruction *I) {
778   if (LoadInst *LI = dyn_cast<LoadInst>(I))
779     return LI->getPointerOperandIndex();
780   if (StoreInst *SI = dyn_cast<StoreInst>(I))
781     return SI->getPointerOperandIndex();
782   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
783     return RMW->getPointerOperandIndex();
784   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
785     return XCHG->getPointerOperandIndex();
786   report_fatal_error("Unexpected instruction");
787   return -1;
788 }
789 
TypeSizeToSizeIndex(uint32_t TypeSize)790 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
791   size_t Res = countTrailingZeros(TypeSize / 8);
792   assert(Res < kNumberOfAccessSizes);
793   return Res;
794 }
795 
untagPointerOperand(Instruction * I,Value * Addr)796 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
797   if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64 ||
798       TargetTriple.isRISCV64())
799     return;
800 
801   IRBuilder<> IRB(I);
802   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
803   Value *UntaggedPtr =
804       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
805   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
806 }
807 
memToShadow(Value * Mem,IRBuilder<> & IRB)808 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
809   // Mem >> Scale
810   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
811   if (Mapping.Offset == 0)
812     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
813   // (Mem >> Scale) + Offset
814   return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
815 }
816 
getAccessInfo(bool IsWrite,unsigned AccessSizeIndex)817 int64_t HWAddressSanitizer::getAccessInfo(bool IsWrite,
818                                           unsigned AccessSizeIndex) {
819   return (CompileKernel << HWASanAccessInfo::CompileKernelShift) |
820          (MatchAllTag.has_value() << HWASanAccessInfo::HasMatchAllShift) |
821          (MatchAllTag.value_or(0) << HWASanAccessInfo::MatchAllShift) |
822          (Recover << HWASanAccessInfo::RecoverShift) |
823          (IsWrite << HWASanAccessInfo::IsWriteShift) |
824          (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
825 }
826 
instrumentMemAccessOutline(Value * Ptr,bool IsWrite,unsigned AccessSizeIndex,Instruction * InsertBefore)827 void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
828                                                     unsigned AccessSizeIndex,
829                                                     Instruction *InsertBefore) {
830   assert(!UsePageAliases);
831   const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
832   IRBuilder<> IRB(InsertBefore);
833   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
834   Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
835   IRB.CreateCall(Intrinsic::getDeclaration(
836                      M, UseShortGranules
837                             ? Intrinsic::hwasan_check_memaccess_shortgranules
838                             : Intrinsic::hwasan_check_memaccess),
839                  {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
840 }
841 
instrumentMemAccessInline(Value * Ptr,bool IsWrite,unsigned AccessSizeIndex,Instruction * InsertBefore)842 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
843                                                    unsigned AccessSizeIndex,
844                                                    Instruction *InsertBefore) {
845   assert(!UsePageAliases);
846   const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
847   IRBuilder<> IRB(InsertBefore);
848 
849   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
850   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, PointerTagShift),
851                                   IRB.getInt8Ty());
852   Value *AddrLong = untagPointer(IRB, PtrLong);
853   Value *Shadow = memToShadow(AddrLong, IRB);
854   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
855   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
856 
857   if (MatchAllTag.has_value()) {
858     Value *TagNotIgnored = IRB.CreateICmpNE(
859         PtrTag, ConstantInt::get(PtrTag->getType(), *MatchAllTag));
860     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
861   }
862 
863   Instruction *CheckTerm =
864       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
865                                 MDBuilder(*C).createBranchWeights(1, 100000));
866 
867   IRB.SetInsertPoint(CheckTerm);
868   Value *OutOfShortGranuleTagRange =
869       IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
870   Instruction *CheckFailTerm =
871       SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
872                                 MDBuilder(*C).createBranchWeights(1, 100000));
873 
874   IRB.SetInsertPoint(CheckTerm);
875   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
876   PtrLowBits = IRB.CreateAdd(
877       PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
878   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
879   SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
880                             MDBuilder(*C).createBranchWeights(1, 100000),
881                             (DomTreeUpdater *)nullptr, nullptr,
882                             CheckFailTerm->getParent());
883 
884   IRB.SetInsertPoint(CheckTerm);
885   Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
886   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
887   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
888   Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
889   SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
890                             MDBuilder(*C).createBranchWeights(1, 100000),
891                             (DomTreeUpdater *)nullptr, nullptr,
892                             CheckFailTerm->getParent());
893 
894   IRB.SetInsertPoint(CheckFailTerm);
895   InlineAsm *Asm;
896   switch (TargetTriple.getArch()) {
897   case Triple::x86_64:
898     // The signal handler will find the data address in rdi.
899     Asm = InlineAsm::get(
900         FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
901         "int3\nnopl " +
902             itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
903             "(%rax)",
904         "{rdi}",
905         /*hasSideEffects=*/true);
906     break;
907   case Triple::aarch64:
908   case Triple::aarch64_be:
909     // The signal handler will find the data address in x0.
910     Asm = InlineAsm::get(
911         FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
912         "brk #" + itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
913         "{x0}",
914         /*hasSideEffects=*/true);
915     break;
916   case Triple::riscv64:
917     // The signal handler will find the data address in x10.
918     Asm = InlineAsm::get(
919         FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
920         "ebreak\naddiw x0, x11, " +
921             itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
922         "{x10}",
923         /*hasSideEffects=*/true);
924     break;
925   default:
926     report_fatal_error("unsupported architecture");
927   }
928   IRB.CreateCall(Asm, PtrLong);
929   if (Recover)
930     cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
931 }
932 
ignoreMemIntrinsic(MemIntrinsic * MI)933 bool HWAddressSanitizer::ignoreMemIntrinsic(MemIntrinsic *MI) {
934   if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) {
935     return (!ClInstrumentWrites || ignoreAccess(MTI, MTI->getDest())) &&
936            (!ClInstrumentReads || ignoreAccess(MTI, MTI->getSource()));
937   }
938   if (isa<MemSetInst>(MI))
939     return !ClInstrumentWrites || ignoreAccess(MI, MI->getDest());
940   return false;
941 }
942 
instrumentMemIntrinsic(MemIntrinsic * MI)943 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
944   IRBuilder<> IRB(MI);
945   if (isa<MemTransferInst>(MI)) {
946     IRB.CreateCall(
947         isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
948         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
949          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
950          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
951   } else if (isa<MemSetInst>(MI)) {
952     IRB.CreateCall(
953         HWAsanMemset,
954         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
955          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
956          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
957   }
958   MI->eraseFromParent();
959 }
960 
instrumentMemAccess(InterestingMemoryOperand & O)961 bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
962   Value *Addr = O.getPtr();
963 
964   LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
965 
966   if (O.MaybeMask)
967     return false; // FIXME
968 
969   IRBuilder<> IRB(O.getInsn());
970   if (isPowerOf2_64(O.TypeSize) &&
971       (O.TypeSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
972       (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
973        *O.Alignment >= O.TypeSize / 8)) {
974     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize);
975     if (InstrumentWithCalls) {
976       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
977                      IRB.CreatePointerCast(Addr, IntptrTy));
978     } else if (OutlinedChecks) {
979       instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
980     } else {
981       instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
982     }
983   } else {
984     IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
985                    {IRB.CreatePointerCast(Addr, IntptrTy),
986                     ConstantInt::get(IntptrTy, O.TypeSize / 8)});
987   }
988   untagPointerOperand(O.getInsn(), Addr);
989 
990   return true;
991 }
992 
tagAlloca(IRBuilder<> & IRB,AllocaInst * AI,Value * Tag,size_t Size)993 void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
994                                    size_t Size) {
995   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
996   if (!UseShortGranules)
997     Size = AlignedSize;
998 
999   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
1000   if (InstrumentWithCalls) {
1001     IRB.CreateCall(HwasanTagMemoryFunc,
1002                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
1003                     ConstantInt::get(IntptrTy, AlignedSize)});
1004   } else {
1005     size_t ShadowSize = Size >> Mapping.Scale;
1006     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
1007     // If this memset is not inlined, it will be intercepted in the hwasan
1008     // runtime library. That's OK, because the interceptor skips the checks if
1009     // the address is in the shadow region.
1010     // FIXME: the interceptor is not as fast as real memset. Consider lowering
1011     // llvm.memset right here into either a sequence of stores, or a call to
1012     // hwasan_tag_memory.
1013     if (ShadowSize)
1014       IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align(1));
1015     if (Size != AlignedSize) {
1016       const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
1017       IRB.CreateStore(ConstantInt::get(Int8Ty, SizeRemainder),
1018                       IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
1019       IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
1020                                    Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
1021                                    AlignedSize - 1));
1022     }
1023   }
1024 }
1025 
retagMask(unsigned AllocaNo)1026 unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
1027   if (TargetTriple.getArch() == Triple::x86_64)
1028     return AllocaNo & TagMaskByte;
1029 
1030   // A list of 8-bit numbers that have at most one run of non-zero bits.
1031   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
1032   // masks.
1033   // The list does not include the value 255, which is used for UAR.
1034   //
1035   // Because we are more likely to use earlier elements of this list than later
1036   // ones, it is sorted in increasing order of probability of collision with a
1037   // mask allocated (temporally) nearby. The program that generated this list
1038   // can be found at:
1039   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
1040   static unsigned FastMasks[] = {0,  128, 64,  192, 32,  96,  224, 112, 240,
1041                                  48, 16,  120, 248, 56,  24,  8,   124, 252,
1042                                  60, 28,  12,  4,   126, 254, 62,  30,  14,
1043                                  6,  2,   127, 63,  31,  15,  7,   3,   1};
1044   return FastMasks[AllocaNo % std::size(FastMasks)];
1045 }
1046 
applyTagMask(IRBuilder<> & IRB,Value * OldTag)1047 Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
1048   if (TargetTriple.getArch() == Triple::x86_64) {
1049     Constant *TagMask = ConstantInt::get(IntptrTy, TagMaskByte);
1050     Value *NewTag = IRB.CreateAnd(OldTag, TagMask);
1051     return NewTag;
1052   }
1053   // aarch64 uses 8-bit tags, so no mask is needed.
1054   return OldTag;
1055 }
1056 
getNextTagWithCall(IRBuilder<> & IRB)1057 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
1058   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
1059 }
1060 
getStackBaseTag(IRBuilder<> & IRB)1061 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
1062   if (ClGenerateTagsWithCalls)
1063     return getNextTagWithCall(IRB);
1064   if (StackBaseTag)
1065     return StackBaseTag;
1066   // Extract some entropy from the stack pointer for the tags.
1067   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
1068   // between functions).
1069   Value *StackPointerLong = getSP(IRB);
1070   Value *StackTag =
1071       applyTagMask(IRB, IRB.CreateXor(StackPointerLong,
1072                                       IRB.CreateLShr(StackPointerLong, 20)));
1073   StackTag->setName("hwasan.stack.base.tag");
1074   return StackTag;
1075 }
1076 
getAllocaTag(IRBuilder<> & IRB,Value * StackTag,AllocaInst * AI,unsigned AllocaNo)1077 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
1078                                         AllocaInst *AI, unsigned AllocaNo) {
1079   if (ClGenerateTagsWithCalls)
1080     return getNextTagWithCall(IRB);
1081   return IRB.CreateXor(StackTag,
1082                        ConstantInt::get(IntptrTy, retagMask(AllocaNo)));
1083 }
1084 
getUARTag(IRBuilder<> & IRB,Value * StackTag)1085 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
1086   if (ClUARRetagToZero)
1087     return ConstantInt::get(IntptrTy, 0);
1088   if (ClGenerateTagsWithCalls)
1089     return getNextTagWithCall(IRB);
1090   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, TagMaskByte));
1091 }
1092 
1093 // Add a tag to an address.
tagPointer(IRBuilder<> & IRB,Type * Ty,Value * PtrLong,Value * Tag)1094 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
1095                                       Value *PtrLong, Value *Tag) {
1096   assert(!UsePageAliases);
1097   Value *TaggedPtrLong;
1098   if (CompileKernel) {
1099     // Kernel addresses have 0xFF in the most significant byte.
1100     Value *ShiftedTag =
1101         IRB.CreateOr(IRB.CreateShl(Tag, PointerTagShift),
1102                      ConstantInt::get(IntptrTy, (1ULL << PointerTagShift) - 1));
1103     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
1104   } else {
1105     // Userspace can simply do OR (tag << PointerTagShift);
1106     Value *ShiftedTag = IRB.CreateShl(Tag, PointerTagShift);
1107     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
1108   }
1109   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
1110 }
1111 
1112 // Remove tag from an address.
untagPointer(IRBuilder<> & IRB,Value * PtrLong)1113 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1114   assert(!UsePageAliases);
1115   Value *UntaggedPtrLong;
1116   if (CompileKernel) {
1117     // Kernel addresses have 0xFF in the most significant byte.
1118     UntaggedPtrLong =
1119         IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(),
1120                                                0xFFULL << PointerTagShift));
1121   } else {
1122     // Userspace addresses have 0x00.
1123     UntaggedPtrLong =
1124         IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
1125                                                 ~(0xFFULL << PointerTagShift)));
1126   }
1127   return UntaggedPtrLong;
1128 }
1129 
getHwasanThreadSlotPtr(IRBuilder<> & IRB,Type * Ty)1130 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
1131   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1132   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
1133     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1134     // in Bionic's libc/private/bionic_tls.h.
1135     Function *ThreadPointerFunc =
1136         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
1137     Value *SlotPtr = IRB.CreatePointerCast(
1138         IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
1139                                IRB.CreateCall(ThreadPointerFunc), 0x30),
1140         Ty->getPointerTo(0));
1141     return SlotPtr;
1142   }
1143   if (ThreadPtrGlobal)
1144     return ThreadPtrGlobal;
1145 
1146   return nullptr;
1147 }
1148 
getPC(IRBuilder<> & IRB)1149 Value *HWAddressSanitizer::getPC(IRBuilder<> &IRB) {
1150   if (TargetTriple.getArch() == Triple::aarch64)
1151     return readRegister(IRB, "pc");
1152   return IRB.CreatePtrToInt(IRB.GetInsertBlock()->getParent(), IntptrTy);
1153 }
1154 
getSP(IRBuilder<> & IRB)1155 Value *HWAddressSanitizer::getSP(IRBuilder<> &IRB) {
1156   if (!CachedSP) {
1157     // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
1158     // first).
1159     Function *F = IRB.GetInsertBlock()->getParent();
1160     Module *M = F->getParent();
1161     auto *GetStackPointerFn = Intrinsic::getDeclaration(
1162         M, Intrinsic::frameaddress,
1163         IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
1164     CachedSP = IRB.CreatePtrToInt(
1165         IRB.CreateCall(GetStackPointerFn,
1166                        {Constant::getNullValue(IRB.getInt32Ty())}),
1167         IntptrTy);
1168   }
1169   return CachedSP;
1170 }
1171 
getFrameRecordInfo(IRBuilder<> & IRB)1172 Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
1173   // Prepare ring buffer data.
1174   Value *PC = getPC(IRB);
1175   Value *SP = getSP(IRB);
1176 
1177   // Mix SP and PC.
1178   // Assumptions:
1179   // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
1180   // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
1181   // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
1182   //       0xSSSSPPPPPPPPPPPP
1183   SP = IRB.CreateShl(SP, 44);
1184   return IRB.CreateOr(PC, SP);
1185 }
1186 
emitPrologue(IRBuilder<> & IRB,bool WithFrameRecord)1187 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1188   if (!Mapping.InTls)
1189     ShadowBase = getShadowNonTls(IRB);
1190   else if (!WithFrameRecord && TargetTriple.isAndroid())
1191     ShadowBase = getDynamicShadowIfunc(IRB);
1192 
1193   if (!WithFrameRecord && ShadowBase)
1194     return;
1195 
1196   Value *SlotPtr = nullptr;
1197   Value *ThreadLong = nullptr;
1198   Value *ThreadLongMaybeUntagged = nullptr;
1199 
1200   auto getThreadLongMaybeUntagged = [&]() {
1201     if (!SlotPtr)
1202       SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
1203     if (!ThreadLong)
1204       ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
1205     // Extract the address field from ThreadLong. Unnecessary on AArch64 with
1206     // TBI.
1207     return TargetTriple.isAArch64() ? ThreadLong
1208                                     : untagPointer(IRB, ThreadLong);
1209   };
1210 
1211   if (WithFrameRecord) {
1212     switch (ClRecordStackHistory) {
1213     case libcall: {
1214       // Emit a runtime call into hwasan rather than emitting instructions for
1215       // recording stack history.
1216       Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1217       IRB.CreateCall(HwasanRecordFrameRecordFunc, {FrameRecordInfo});
1218       break;
1219     }
1220     case instr: {
1221       ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1222 
1223       StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
1224 
1225       // Store data to ring buffer.
1226       Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1227       Value *RecordPtr = IRB.CreateIntToPtr(ThreadLongMaybeUntagged,
1228                                             IntptrTy->getPointerTo(0));
1229       IRB.CreateStore(FrameRecordInfo, RecordPtr);
1230 
1231       // Update the ring buffer. Top byte of ThreadLong defines the size of the
1232       // buffer in pages, it must be a power of two, and the start of the buffer
1233       // must be aligned by twice that much. Therefore wrap around of the ring
1234       // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
1235       // The use of AShr instead of LShr is due to
1236       //   https://bugs.llvm.org/show_bug.cgi?id=39030
1237       // Runtime library makes sure not to use the highest bit.
1238       Value *WrapMask = IRB.CreateXor(
1239           IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
1240           ConstantInt::get(IntptrTy, (uint64_t)-1));
1241       Value *ThreadLongNew = IRB.CreateAnd(
1242           IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
1243       IRB.CreateStore(ThreadLongNew, SlotPtr);
1244       break;
1245     }
1246     case none: {
1247       llvm_unreachable(
1248           "A stack history recording mode should've been selected.");
1249     }
1250     }
1251   }
1252 
1253   if (!ShadowBase) {
1254     if (!ThreadLongMaybeUntagged)
1255       ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1256 
1257     // Get shadow base address by aligning RecordPtr up.
1258     // Note: this is not correct if the pointer is already aligned.
1259     // Runtime library will make sure this never happens.
1260     ShadowBase = IRB.CreateAdd(
1261         IRB.CreateOr(
1262             ThreadLongMaybeUntagged,
1263             ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1264         ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1265     ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
1266   }
1267 }
1268 
readRegister(IRBuilder<> & IRB,StringRef Name)1269 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1270   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1271   Function *ReadRegister =
1272       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1273   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1274   Value *Args[] = {MetadataAsValue::get(*C, MD)};
1275   return IRB.CreateCall(ReadRegister, Args);
1276 }
1277 
instrumentLandingPads(SmallVectorImpl<Instruction * > & LandingPadVec)1278 bool HWAddressSanitizer::instrumentLandingPads(
1279     SmallVectorImpl<Instruction *> &LandingPadVec) {
1280   for (auto *LP : LandingPadVec) {
1281     IRBuilder<> IRB(LP->getNextNode());
1282     IRB.CreateCall(
1283         HWAsanHandleVfork,
1284         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1285                                                                       : "sp")});
1286   }
1287   return true;
1288 }
1289 
isLifetimeIntrinsic(Value * V)1290 static bool isLifetimeIntrinsic(Value *V) {
1291   auto *II = dyn_cast<IntrinsicInst>(V);
1292   return II && II->isLifetimeStartOrEnd();
1293 }
1294 
instrumentStack(memtag::StackInfo & SInfo,Value * StackTag,const DominatorTree & DT,const PostDominatorTree & PDT,const LoopInfo & LI)1295 bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
1296                                          Value *StackTag,
1297                                          const DominatorTree &DT,
1298                                          const PostDominatorTree &PDT,
1299                                          const LoopInfo &LI) {
1300   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1301   // alloca addresses using that. Unfortunately, offsets are not known yet
1302   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1303   // temp, shift-OR it into each alloca address and xor with the retag mask.
1304   // This generates one extra instruction per alloca use.
1305   unsigned int I = 0;
1306 
1307   for (auto &KV : SInfo.AllocasToInstrument) {
1308     auto N = I++;
1309     auto *AI = KV.first;
1310     memtag::AllocaInfo &Info = KV.second;
1311     IRBuilder<> IRB(AI->getNextNode());
1312 
1313     // Replace uses of the alloca with tagged address.
1314     Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
1315     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1316     Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
1317     std::string Name =
1318         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1319     Replacement->setName(Name + ".hwasan");
1320 
1321     size_t Size = memtag::getAllocaSizeInBytes(*AI);
1322     size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1323 
1324     Value *AICast = IRB.CreatePointerCast(AI, Int8PtrTy);
1325 
1326     auto HandleLifetime = [&](IntrinsicInst *II) {
1327       // Set the lifetime intrinsic to cover the whole alloca. This reduces the
1328       // set of assumptions we need to make about the lifetime. Without this we
1329       // would need to ensure that we can track the lifetime pointer to a
1330       // constant offset from the alloca, and would still need to change the
1331       // size to include the extra alignment we use for the untagging to make
1332       // the size consistent.
1333       //
1334       // The check for standard lifetime below makes sure that we have exactly
1335       // one set of start / end in any execution (i.e. the ends are not
1336       // reachable from each other), so this will not cause any problems.
1337       II->setArgOperand(0, ConstantInt::get(Int64Ty, AlignedSize));
1338       II->setArgOperand(1, AICast);
1339     };
1340     llvm::for_each(Info.LifetimeStart, HandleLifetime);
1341     llvm::for_each(Info.LifetimeEnd, HandleLifetime);
1342 
1343     AI->replaceUsesWithIf(Replacement, [AICast, AILong](Use &U) {
1344       auto *User = U.getUser();
1345       return User != AILong && User != AICast && !isLifetimeIntrinsic(User);
1346     });
1347 
1348     for (auto *DDI : Info.DbgVariableIntrinsics) {
1349       // Prepend "tag_offset, N" to the dwarf expression.
1350       // Tag offset logically applies to the alloca pointer, and it makes sense
1351       // to put it at the beginning of the expression.
1352       SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset,
1353                                          retagMask(N)};
1354       for (size_t LocNo = 0; LocNo < DDI->getNumVariableLocationOps(); ++LocNo)
1355         if (DDI->getVariableLocationOp(LocNo) == AI)
1356           DDI->setExpression(DIExpression::appendOpsToArg(DDI->getExpression(),
1357                                                           NewOps, LocNo));
1358     }
1359 
1360     auto TagEnd = [&](Instruction *Node) {
1361       IRB.SetInsertPoint(Node);
1362       Value *UARTag = getUARTag(IRB, StackTag);
1363       // When untagging, use the `AlignedSize` because we need to set the tags
1364       // for the entire alloca to zero. If we used `Size` here, we would
1365       // keep the last granule tagged, and store zero in the last byte of the
1366       // last granule, due to how short granules are implemented.
1367       tagAlloca(IRB, AI, UARTag, AlignedSize);
1368     };
1369     // Calls to functions that may return twice (e.g. setjmp) confuse the
1370     // postdominator analysis, and will leave us to keep memory tagged after
1371     // function return. Work around this by always untagging at every return
1372     // statement if return_twice functions are called.
1373     bool StandardLifetime =
1374         SInfo.UnrecognizedLifetimes.empty() &&
1375         memtag::isStandardLifetime(Info.LifetimeStart, Info.LifetimeEnd, &DT,
1376                                    &LI, ClMaxLifetimes) &&
1377         !SInfo.CallsReturnTwice;
1378     if (DetectUseAfterScope && StandardLifetime) {
1379       IntrinsicInst *Start = Info.LifetimeStart[0];
1380       IRB.SetInsertPoint(Start->getNextNode());
1381       tagAlloca(IRB, AI, Tag, Size);
1382       if (!memtag::forAllReachableExits(DT, PDT, LI, Start, Info.LifetimeEnd,
1383                                         SInfo.RetVec, TagEnd)) {
1384         for (auto *End : Info.LifetimeEnd)
1385           End->eraseFromParent();
1386       }
1387     } else {
1388       tagAlloca(IRB, AI, Tag, Size);
1389       for (auto *RI : SInfo.RetVec)
1390         TagEnd(RI);
1391       // We inserted tagging outside of the lifetimes, so we have to remove
1392       // them.
1393       for (auto &II : Info.LifetimeStart)
1394         II->eraseFromParent();
1395       for (auto &II : Info.LifetimeEnd)
1396         II->eraseFromParent();
1397     }
1398     memtag::alignAndPadAlloca(Info, Mapping.getObjectAlignment());
1399   }
1400   for (auto &I : SInfo.UnrecognizedLifetimes)
1401     I->eraseFromParent();
1402   return true;
1403 }
1404 
sanitizeFunction(Function & F,FunctionAnalysisManager & FAM)1405 bool HWAddressSanitizer::sanitizeFunction(Function &F,
1406                                           FunctionAnalysisManager &FAM) {
1407   if (&F == HwasanCtorFunction)
1408     return false;
1409 
1410   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1411     return false;
1412 
1413   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1414 
1415   SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1416   SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1417   SmallVector<Instruction *, 8> LandingPadVec;
1418 
1419   memtag::StackInfoBuilder SIB(SSI);
1420   for (auto &Inst : instructions(F)) {
1421     if (InstrumentStack) {
1422       SIB.visit(Inst);
1423     }
1424 
1425     if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1426       LandingPadVec.push_back(&Inst);
1427 
1428     getInterestingMemoryOperands(&Inst, OperandsToInstrument);
1429 
1430     if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
1431       if (!ignoreMemIntrinsic(MI))
1432         IntrinToInstrument.push_back(MI);
1433   }
1434 
1435   memtag::StackInfo &SInfo = SIB.get();
1436 
1437   initializeCallbacks(*F.getParent());
1438 
1439   bool Changed = false;
1440 
1441   if (!LandingPadVec.empty())
1442     Changed |= instrumentLandingPads(LandingPadVec);
1443 
1444   if (SInfo.AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1445       F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1446     // __hwasan_personality_thunk is a no-op for functions without an
1447     // instrumented stack, so we can drop it.
1448     F.setPersonalityFn(nullptr);
1449     Changed = true;
1450   }
1451 
1452   if (SInfo.AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1453       IntrinToInstrument.empty())
1454     return Changed;
1455 
1456   assert(!ShadowBase);
1457 
1458   Instruction *InsertPt = &*F.getEntryBlock().begin();
1459   IRBuilder<> EntryIRB(InsertPt);
1460   emitPrologue(EntryIRB,
1461                /*WithFrameRecord*/ ClRecordStackHistory != none &&
1462                    Mapping.WithFrameRecord &&
1463                    !SInfo.AllocasToInstrument.empty());
1464 
1465   if (!SInfo.AllocasToInstrument.empty()) {
1466     const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
1467     const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(F);
1468     const LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
1469     Value *StackTag =
1470         ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1471     instrumentStack(SInfo, StackTag, DT, PDT, LI);
1472   }
1473 
1474   // If we split the entry block, move any allocas that were originally in the
1475   // entry block back into the entry block so that they aren't treated as
1476   // dynamic allocas.
1477   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1478     InsertPt = &*F.getEntryBlock().begin();
1479     for (Instruction &I :
1480          llvm::make_early_inc_range(*EntryIRB.GetInsertBlock())) {
1481       if (auto *AI = dyn_cast<AllocaInst>(&I))
1482         if (isa<ConstantInt>(AI->getArraySize()))
1483           I.moveBefore(InsertPt);
1484     }
1485   }
1486 
1487   for (auto &Operand : OperandsToInstrument)
1488     instrumentMemAccess(Operand);
1489 
1490   if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1491     for (auto *Inst : IntrinToInstrument)
1492       instrumentMemIntrinsic(Inst);
1493   }
1494 
1495   ShadowBase = nullptr;
1496   StackBaseTag = nullptr;
1497   CachedSP = nullptr;
1498 
1499   return true;
1500 }
1501 
instrumentGlobal(GlobalVariable * GV,uint8_t Tag)1502 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1503   assert(!UsePageAliases);
1504   Constant *Initializer = GV->getInitializer();
1505   uint64_t SizeInBytes =
1506       M.getDataLayout().getTypeAllocSize(Initializer->getType());
1507   uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1508   if (SizeInBytes != NewSize) {
1509     // Pad the initializer out to the next multiple of 16 bytes and add the
1510     // required short granule tag.
1511     std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1512     Init.back() = Tag;
1513     Constant *Padding = ConstantDataArray::get(*C, Init);
1514     Initializer = ConstantStruct::getAnon({Initializer, Padding});
1515   }
1516 
1517   auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1518                                    GlobalValue::ExternalLinkage, Initializer,
1519                                    GV->getName() + ".hwasan");
1520   NewGV->copyAttributesFrom(GV);
1521   NewGV->setLinkage(GlobalValue::PrivateLinkage);
1522   NewGV->copyMetadata(GV, 0);
1523   NewGV->setAlignment(
1524       std::max(GV->getAlign().valueOrOne(), Mapping.getObjectAlignment()));
1525 
1526   // It is invalid to ICF two globals that have different tags. In the case
1527   // where the size of the global is a multiple of the tag granularity the
1528   // contents of the globals may be the same but the tags (i.e. symbol values)
1529   // may be different, and the symbols are not considered during ICF. In the
1530   // case where the size is not a multiple of the granularity, the short granule
1531   // tags would discriminate two globals with different tags, but there would
1532   // otherwise be nothing stopping such a global from being incorrectly ICF'd
1533   // with an uninstrumented (i.e. tag 0) global that happened to have the short
1534   // granule tag in the last byte.
1535   NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1536 
1537   // Descriptor format (assuming little-endian):
1538   // bytes 0-3: relative address of global
1539   // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1540   // it isn't, we create multiple descriptors)
1541   // byte 7: tag
1542   auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1543   const uint64_t MaxDescriptorSize = 0xfffff0;
1544   for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1545        DescriptorPos += MaxDescriptorSize) {
1546     auto *Descriptor =
1547         new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1548                            nullptr, GV->getName() + ".hwasan.descriptor");
1549     auto *GVRelPtr = ConstantExpr::getTrunc(
1550         ConstantExpr::getAdd(
1551             ConstantExpr::getSub(
1552                 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1553                 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1554             ConstantInt::get(Int64Ty, DescriptorPos)),
1555         Int32Ty);
1556     uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1557     auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1558     Descriptor->setComdat(NewGV->getComdat());
1559     Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1560     Descriptor->setSection("hwasan_globals");
1561     Descriptor->setMetadata(LLVMContext::MD_associated,
1562                             MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1563     appendToCompilerUsed(M, Descriptor);
1564   }
1565 
1566   Constant *Aliasee = ConstantExpr::getIntToPtr(
1567       ConstantExpr::getAdd(
1568           ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1569           ConstantInt::get(Int64Ty, uint64_t(Tag) << PointerTagShift)),
1570       GV->getType());
1571   auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1572                                     GV->getLinkage(), "", Aliasee, &M);
1573   Alias->setVisibility(GV->getVisibility());
1574   Alias->takeName(GV);
1575   GV->replaceAllUsesWith(Alias);
1576   GV->eraseFromParent();
1577 }
1578 
instrumentGlobals()1579 void HWAddressSanitizer::instrumentGlobals() {
1580   std::vector<GlobalVariable *> Globals;
1581   for (GlobalVariable &GV : M.globals()) {
1582     if (GV.hasSanitizerMetadata() && GV.getSanitizerMetadata().NoHWAddress)
1583       continue;
1584 
1585     if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1586         GV.isThreadLocal())
1587       continue;
1588 
1589     // Common symbols can't have aliases point to them, so they can't be tagged.
1590     if (GV.hasCommonLinkage())
1591       continue;
1592 
1593     // Globals with custom sections may be used in __start_/__stop_ enumeration,
1594     // which would be broken both by adding tags and potentially by the extra
1595     // padding/alignment that we insert.
1596     if (GV.hasSection())
1597       continue;
1598 
1599     Globals.push_back(&GV);
1600   }
1601 
1602   MD5 Hasher;
1603   Hasher.update(M.getSourceFileName());
1604   MD5::MD5Result Hash;
1605   Hasher.final(Hash);
1606   uint8_t Tag = Hash[0];
1607 
1608   for (GlobalVariable *GV : Globals) {
1609     Tag &= TagMaskByte;
1610     // Skip tag 0 in order to avoid collisions with untagged memory.
1611     if (Tag == 0)
1612       Tag = 1;
1613     instrumentGlobal(GV, Tag++);
1614   }
1615 }
1616 
instrumentPersonalityFunctions()1617 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1618   // We need to untag stack frames as we unwind past them. That is the job of
1619   // the personality function wrapper, which either wraps an existing
1620   // personality function or acts as a personality function on its own. Each
1621   // function that has a personality function or that can be unwound past has
1622   // its personality function changed to a thunk that calls the personality
1623   // function wrapper in the runtime.
1624   MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1625   for (Function &F : M) {
1626     if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1627       continue;
1628 
1629     if (F.hasPersonalityFn()) {
1630       PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1631     } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1632       PersonalityFns[nullptr].push_back(&F);
1633     }
1634   }
1635 
1636   if (PersonalityFns.empty())
1637     return;
1638 
1639   FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1640       "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1641       Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1642   FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1643   FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1644 
1645   for (auto &P : PersonalityFns) {
1646     std::string ThunkName = kHwasanPersonalityThunkName;
1647     if (P.first)
1648       ThunkName += ("." + P.first->getName()).str();
1649     FunctionType *ThunkFnTy = FunctionType::get(
1650         Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1651     bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1652                                cast<GlobalValue>(P.first)->hasLocalLinkage());
1653     auto *ThunkFn = Function::Create(ThunkFnTy,
1654                                      IsLocal ? GlobalValue::InternalLinkage
1655                                              : GlobalValue::LinkOnceODRLinkage,
1656                                      ThunkName, &M);
1657     if (!IsLocal) {
1658       ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1659       ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1660     }
1661 
1662     auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1663     IRBuilder<> IRB(BB);
1664     CallInst *WrapperCall = IRB.CreateCall(
1665         HwasanPersonalityWrapper,
1666         {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1667          ThunkFn->getArg(3), ThunkFn->getArg(4),
1668          P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1669                  : Constant::getNullValue(Int8PtrTy),
1670          IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1671          IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1672     WrapperCall->setTailCall();
1673     IRB.CreateRet(WrapperCall);
1674 
1675     for (Function *F : P.second)
1676       F->setPersonalityFn(ThunkFn);
1677   }
1678 }
1679 
init(Triple & TargetTriple,bool InstrumentWithCalls)1680 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
1681                                              bool InstrumentWithCalls) {
1682   Scale = kDefaultShadowScale;
1683   if (TargetTriple.isOSFuchsia()) {
1684     // Fuchsia is always PIE, which means that the beginning of the address
1685     // space is always available.
1686     InGlobal = false;
1687     InTls = false;
1688     Offset = 0;
1689     WithFrameRecord = true;
1690   } else if (ClMappingOffset.getNumOccurrences() > 0) {
1691     InGlobal = false;
1692     InTls = false;
1693     Offset = ClMappingOffset;
1694     WithFrameRecord = false;
1695   } else if (ClEnableKhwasan || InstrumentWithCalls) {
1696     InGlobal = false;
1697     InTls = false;
1698     Offset = 0;
1699     WithFrameRecord = false;
1700   } else if (ClWithIfunc) {
1701     InGlobal = true;
1702     InTls = false;
1703     Offset = kDynamicShadowSentinel;
1704     WithFrameRecord = false;
1705   } else if (ClWithTls) {
1706     InGlobal = false;
1707     InTls = true;
1708     Offset = kDynamicShadowSentinel;
1709     WithFrameRecord = true;
1710   } else {
1711     InGlobal = false;
1712     InTls = false;
1713     Offset = kDynamicShadowSentinel;
1714     WithFrameRecord = false;
1715   }
1716 }
1717