1 // Copyright 2020 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "LLVMReactor.hpp"
16
17 #include "Debug.hpp"
18 #include "ExecutableMemory.hpp"
19 #include "LLVMAsm.hpp"
20 #include "PragmaInternals.hpp"
21 #include "Routine.hpp"
22
23 // TODO(b/143539525): Eliminate when warning has been fixed.
24 #ifdef _MSC_VER
25 __pragma(warning(push))
26 __pragma(warning(disable : 4146)) // unary minus operator applied to unsigned type, result still unsigned
27 #endif
28
29 // See https://groups.google.com/g/llvm-dev/c/CAE7Va57h2c/m/74ITeXFEAQAJ
30 // for information about `RTDyldObjectLinkingLayer` vs `ObjectLinkingLayer`.
31 // On RISC-V and LoongArch, only `ObjectLinkingLayer` is supported.
32 #if defined(__riscv) || defined(__loongarch__)
33 #define USE_LEGACY_OBJECT_LINKING_LAYER 0
34 #else
35 #define USE_LEGACY_OBJECT_LINKING_LAYER 1
36 #endif
37
38 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
39 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
40
41 #if USE_LEGACY_OBJECT_LINKING_LAYER
42 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
43 #else
44 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
45 #endif
46 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Verifier.h"
49 #include "llvm/Support/CommandLine.h"
50 #include "llvm/Support/Host.h"
51 #include "llvm/Support/TargetSelect.h"
52 #include "llvm/Transforms/InstCombine/InstCombine.h"
53 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
54 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
55 #include "llvm/Transforms/Scalar.h"
56 #include "llvm/Transforms/Scalar/GVN.h"
57
58 #if LLVM_VERSION_MAJOR >= 13 // New pass manager
59 # include "llvm/IR/PassManager.h"
60 # include "llvm/Passes/PassBuilder.h"
61 # include "llvm/Transforms/Scalar/ADCE.h"
62 # include "llvm/Transforms/Scalar/DeadStoreElimination.h"
63 # include "llvm/Transforms/Scalar/EarlyCSE.h"
64 # include "llvm/Transforms/Scalar/LICM.h"
65 # include "llvm/Transforms/Scalar/Reassociate.h"
66 # include "llvm/Transforms/Scalar/SCCP.h"
67 # include "llvm/Transforms/Scalar/SROA.h"
68 # include "llvm/Transforms/Scalar/SimplifyCFG.h"
69 #else // Legacy pass manager
70 # include "llvm/IR/LegacyPassManager.h"
71 # include "llvm/Pass.h"
72 # include "llvm/Transforms/Coroutines.h"
73 # include "llvm/Transforms/IPO.h"
74 #endif
75
76 #ifdef _MSC_VER
77 __pragma(warning(pop))
78 #endif
79
80 #if defined(__unix__) || defined(__APPLE__) || defined(__Fuchsia__)
81 # define ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED true
82 # if __has_feature(memory_sanitizer) || __has_feature(address_sanitizer)
83 # include <dlfcn.h> // dlsym()
84 # endif
85 #else
86 # define ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED false
87 #endif
88
89 #ifndef REACTOR_ASM_EMIT_DIR
90 # define REACTOR_ASM_EMIT_DIR "./"
91 #endif
92
93 #if defined(_WIN64)
94 extern "C" void __chkstk();
95 #elif defined(_WIN32)
96 extern "C" void _chkstk();
97 #endif
98
99 #ifdef __ARM_EABI__
100 extern "C" signed __aeabi_idivmod();
101 #endif
102
103 #if __has_feature(memory_sanitizer)
104 # include "sanitizer/msan_interface.h"
105
106 // MemorySanitizer uses thread-local storage (TLS) data arrays for passing around
107 // the 'shadow' values of function arguments and return values. The LLVM JIT can't
108 // access TLS directly, but it calls __emutls_get_address() to obtain the address.
109 // Typically, it would be passed a pointer to an __emutls_control structure with a
110 // name starting with "__emutls_v." that represents the TLS. Both the address of
111 // __emutls_get_address and the __emutls_v. structures are provided to the JIT by
112 // the symbol resolver, which can be overridden.
113 // We take advantage of this by substituting __emutls_get_address() with our own
114 // implementation, namely rr::getTLSAddress(), and substituting the __emutls_v
115 // variables with rr::MSanTLS enums. getTLSAddress() can then provide the address
116 // of the real TLS variable corresponding to the enum, in statically compiled C++.
117
118 // Forward declare the real TLS variables used by MemorySanitizer. These are
119 // defined in llvm-project/compiler-rt/lib/msan/msan.cpp.
120 extern __thread unsigned long long __msan_param_tls[];
121 extern __thread unsigned int __msan_param_origin_tls[];
122 extern __thread unsigned long long __msan_retval_tls[];
123 extern __thread unsigned int __msan_retval_origin_tls;
124 extern __thread unsigned long long __msan_va_arg_tls[];
125 extern __thread unsigned int __msan_va_arg_origin_tls[];
126 extern __thread unsigned long long __msan_va_arg_overflow_size_tls;
127 extern __thread unsigned int __msan_origin_tls;
128
129 namespace rr {
130
131 enum class MSanTLS
132 {
133 param = 1, // __msan_param_tls
134 param_origin, //__msan_param_origin_tls
135 retval, // __msan_retval_tls
136 retval_origin, //__msan_retval_origin_tls
137 va_arg, // __msan_va_arg_tls
138 va_arg_origin, // __msan_va_arg_origin_tls
139 va_arg_overflow_size, // __msan_va_arg_overflow_size_tls
140 origin, //__msan_origin_tls
141 };
142
getTLSAddress(void * control)143 static void *getTLSAddress(void *control)
144 {
145 auto tlsIndex = static_cast<MSanTLS>(reinterpret_cast<uintptr_t>(control));
146 switch(tlsIndex)
147 {
148 case MSanTLS::param: return reinterpret_cast<void *>(&__msan_param_tls);
149 case MSanTLS::param_origin: return reinterpret_cast<void *>(&__msan_param_origin_tls);
150 case MSanTLS::retval: return reinterpret_cast<void *>(&__msan_retval_tls);
151 case MSanTLS::retval_origin: return reinterpret_cast<void *>(&__msan_retval_origin_tls);
152 case MSanTLS::va_arg: return reinterpret_cast<void *>(&__msan_va_arg_tls);
153 case MSanTLS::va_arg_origin: return reinterpret_cast<void *>(&__msan_va_arg_origin_tls);
154 case MSanTLS::va_arg_overflow_size: return reinterpret_cast<void *>(&__msan_va_arg_overflow_size_tls);
155 case MSanTLS::origin: return reinterpret_cast<void *>(&__msan_origin_tls);
156
157 default:
158 UNSUPPORTED("MemorySanitizer used an unrecognized TLS variable: %d", static_cast<int>(tlsIndex));
159 return nullptr;
160 }
161 }
162
163 } // namespace rr
164 #endif
165
166 namespace {
167
168 // TODO(b/174587935): Eliminate command-line parsing.
parseCommandLineOptionsOnce(int argc,const char * const * argv)169 bool parseCommandLineOptionsOnce(int argc, const char *const *argv)
170 {
171 // Use a static immediately invoked lambda to make this thread safe
172 static auto initialized = [=]() {
173 return llvm::cl::ParseCommandLineOptions(argc, argv);
174 }();
175
176 return initialized;
177 }
178
179 // JITGlobals is a singleton that holds all the immutable machine specific
180 // information for the host device.
181 class JITGlobals
182 {
183 public:
184 static JITGlobals *get();
185
186 llvm::orc::JITTargetMachineBuilder getTargetMachineBuilder() const;
187 const llvm::DataLayout &getDataLayout() const;
188 const llvm::Triple &getTargetTriple() const;
189
190 private:
191 JITGlobals(llvm::orc::JITTargetMachineBuilder &&jitTargetMachineBuilder, llvm::DataLayout &&dataLayout);
192
193 static llvm::CodeGenOpt::Level toLLVM(int level);
194
195 const llvm::orc::JITTargetMachineBuilder jitTargetMachineBuilder;
196 const llvm::DataLayout dataLayout;
197 };
198
get()199 JITGlobals *JITGlobals::get()
200 {
201 static JITGlobals instance = [] {
202 const char *argv[] = {
203 "Reactor",
204 #if defined(__i386__) || defined(__x86_64__)
205 "-x86-asm-syntax=intel", // Use Intel syntax rather than the default AT&T
206 #endif
207 #if LLVM_VERSION_MAJOR <= 12
208 "-warn-stack-size=524288", // Warn when a function uses more than 512 KiB of stack memory
209 #endif
210 };
211
212 parseCommandLineOptionsOnce(sizeof(argv) / sizeof(argv[0]), argv);
213
214 llvm::InitializeNativeTarget();
215 llvm::InitializeNativeTargetAsmPrinter();
216 llvm::InitializeNativeTargetAsmParser();
217
218 // TODO(b/171236524): JITTargetMachineBuilder::detectHost() currently uses the target triple of the host,
219 // rather than a valid triple for the current process. Once fixed, we can use that function instead.
220 llvm::orc::JITTargetMachineBuilder jitTargetMachineBuilder(llvm::Triple(LLVM_DEFAULT_TARGET_TRIPLE));
221
222 // Retrieve host CPU name and sub-target features and add them to builder.
223 // Relocation model, code model and codegen opt level are kept to default values.
224 llvm::StringMap<bool> cpuFeatures;
225 bool ok = llvm::sys::getHostCPUFeatures(cpuFeatures);
226
227 #if defined(__i386__) || defined(__x86_64__) || \
228 (defined(__linux__) && (defined(__arm__) || defined(__aarch64__)))
229 ASSERT_MSG(ok, "llvm::sys::getHostCPUFeatures returned false");
230 #else
231 (void)ok; // getHostCPUFeatures always returns false on other platforms
232 #endif
233 for(auto &feature : cpuFeatures)
234 {
235 jitTargetMachineBuilder.getFeatures().AddFeature(feature.first(), feature.second);
236 }
237
238 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
239
240 #if defined(__riscv) && __riscv_xlen == 64
241 // jitTargetMachineBuilder.getFeatures() on RISC-V does
242 // not return the RISC-V CPU extensions, so they are
243 // manually added.
244 jitTargetMachineBuilder.getFeatures().AddFeature("+m");
245 jitTargetMachineBuilder.getFeatures().AddFeature("+a");
246 jitTargetMachineBuilder.getFeatures().AddFeature("+f");
247 jitTargetMachineBuilder.getFeatures().AddFeature("+d");
248 jitTargetMachineBuilder.getFeatures().AddFeature("+c");
249 // The default code model is "Small".
250 // On RISC-V, using the default code model results in an
251 // "Unsupported riscv relocation" error.
252 jitTargetMachineBuilder.setCodeModel(llvm::CodeModel::Medium);
253 #elif defined(__loongarch__)
254 // jitTargetMachineBuilder.getFeatures() on LoongArch does
255 // not return the LoongArch CPU extensions, so they are
256 // manually added.
257 jitTargetMachineBuilder.getFeatures().AddFeature("+d");
258 #endif
259
260 jitTargetMachineBuilder.setCPU(std::string(llvm::sys::getHostCPUName()));
261 #else
262 jitTargetMachineBuilder.setCPU(llvm::sys::getHostCPUName());
263 #endif
264
265 // Reactor's MemorySanitizer support depends on intercepting __emutls_get_address calls.
266 #if LLVM_VERSION_MAJOR < 17
267 ASSERT(!__has_feature(memory_sanitizer) || (jitTargetMachineBuilder.getOptions().ExplicitEmulatedTLS &&
268 jitTargetMachineBuilder.getOptions().EmulatedTLS));
269 #else
270 ASSERT(!__has_feature(memory_sanitizer) || jitTargetMachineBuilder.getOptions().EmulatedTLS);
271 #endif
272
273 auto dataLayout = jitTargetMachineBuilder.getDefaultDataLayoutForTarget();
274 ASSERT_MSG(dataLayout, "JITTargetMachineBuilder::getDefaultDataLayoutForTarget() failed");
275
276 return JITGlobals(std::move(jitTargetMachineBuilder), std::move(dataLayout.get()));
277 }();
278
279 return &instance;
280 }
281
getTargetMachineBuilder() const282 llvm::orc::JITTargetMachineBuilder JITGlobals::getTargetMachineBuilder() const
283 {
284 llvm::orc::JITTargetMachineBuilder out = jitTargetMachineBuilder;
285 out.setCodeGenOptLevel(toLLVM(rr::getPragmaState(rr::OptimizationLevel)));
286
287 return out;
288 }
289
getDataLayout() const290 const llvm::DataLayout &JITGlobals::getDataLayout() const
291 {
292 return dataLayout;
293 }
294
getTargetTriple() const295 const llvm::Triple &JITGlobals::getTargetTriple() const
296 {
297 return jitTargetMachineBuilder.getTargetTriple();
298 }
299
JITGlobals(llvm::orc::JITTargetMachineBuilder && jitTargetMachineBuilder,llvm::DataLayout && dataLayout)300 JITGlobals::JITGlobals(llvm::orc::JITTargetMachineBuilder &&jitTargetMachineBuilder, llvm::DataLayout &&dataLayout)
301 : jitTargetMachineBuilder(jitTargetMachineBuilder)
302 , dataLayout(dataLayout)
303 {
304 }
305
toLLVM(int level)306 llvm::CodeGenOpt::Level JITGlobals::toLLVM(int level)
307 {
308 // TODO(b/173257647): MemorySanitizer instrumentation produces IR which takes
309 // a lot longer to process by the machine code optimization passes. Disabling
310 // them has a negligible effect on code quality but compiles much faster.
311 if(__has_feature(memory_sanitizer))
312 {
313 return llvm::CodeGenOpt::None;
314 }
315
316 switch(level)
317 {
318 case 0: return llvm::CodeGenOpt::None;
319 case 1: return llvm::CodeGenOpt::Less;
320 case 2: return llvm::CodeGenOpt::Default;
321 case 3: return llvm::CodeGenOpt::Aggressive;
322 default: UNREACHABLE("Unknown Optimization Level %d", int(level));
323 }
324
325 return llvm::CodeGenOpt::Default;
326 }
327
328 class MemoryMapper final : public llvm::SectionMemoryManager::MemoryMapper
329 {
330 public:
MemoryMapper()331 MemoryMapper() {}
~MemoryMapper()332 ~MemoryMapper() final {}
333
allocateMappedMemory(llvm::SectionMemoryManager::AllocationPurpose purpose,size_t numBytes,const llvm::sys::MemoryBlock * const nearBlock,unsigned flags,std::error_code & errorCode)334 llvm::sys::MemoryBlock allocateMappedMemory(
335 llvm::SectionMemoryManager::AllocationPurpose purpose,
336 size_t numBytes, const llvm::sys::MemoryBlock *const nearBlock,
337 unsigned flags, std::error_code &errorCode) final
338 {
339 errorCode = std::error_code();
340
341 // Round up numBytes to page size.
342 size_t pageSize = rr::memoryPageSize();
343 numBytes = (numBytes + pageSize - 1) & ~(pageSize - 1);
344
345 bool need_exec =
346 purpose == llvm::SectionMemoryManager::AllocationPurpose::Code;
347 void *addr = rr::allocateMemoryPages(
348 numBytes, flagsToPermissions(flags), need_exec);
349 if(!addr)
350 return llvm::sys::MemoryBlock();
351 return llvm::sys::MemoryBlock(addr, numBytes);
352 }
353
protectMappedMemory(const llvm::sys::MemoryBlock & block,unsigned flags)354 std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &block,
355 unsigned flags)
356 {
357 // Round down base address to align with a page boundary. This matches
358 // DefaultMMapper behavior.
359 void *addr = block.base();
360 size_t size = block.allocatedSize();
361 size_t pageSize = rr::memoryPageSize();
362 addr = reinterpret_cast<void *>(
363 reinterpret_cast<uintptr_t>(addr) & ~(pageSize - 1));
364 size += reinterpret_cast<uintptr_t>(block.base()) -
365 reinterpret_cast<uintptr_t>(addr);
366
367 rr::protectMemoryPages(addr, size, flagsToPermissions(flags));
368 return std::error_code();
369 }
370
releaseMappedMemory(llvm::sys::MemoryBlock & block)371 std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &block)
372 {
373 size_t size = block.allocatedSize();
374
375 rr::deallocateMemoryPages(block.base(), size);
376 return std::error_code();
377 }
378
379 private:
flagsToPermissions(unsigned flags)380 int flagsToPermissions(unsigned flags)
381 {
382 int result = 0;
383 if(flags & llvm::sys::Memory::MF_READ)
384 {
385 result |= rr::PERMISSION_READ;
386 }
387 if(flags & llvm::sys::Memory::MF_WRITE)
388 {
389 result |= rr::PERMISSION_WRITE;
390 }
391 if(flags & llvm::sys::Memory::MF_EXEC)
392 {
393 result |= rr::PERMISSION_EXECUTE;
394 }
395 return result;
396 }
397 };
398
399 template<typename T>
alignUp(T val,T alignment)400 T alignUp(T val, T alignment)
401 {
402 return alignment * ((val + alignment - 1) / alignment);
403 }
404
alignedAlloc(size_t size,size_t alignment)405 void *alignedAlloc(size_t size, size_t alignment)
406 {
407 ASSERT(alignment < 256);
408 auto allocation = new uint8_t[size + sizeof(uint8_t) + alignment];
409 auto aligned = allocation;
410 aligned += sizeof(uint8_t); // Make space for the base-address offset.
411 aligned = reinterpret_cast<uint8_t *>(alignUp(reinterpret_cast<uintptr_t>(aligned), alignment)); // align
412 auto offset = static_cast<uint8_t>(aligned - allocation);
413 aligned[-1] = offset;
414 return aligned;
415 }
416
alignedFree(void * ptr)417 void alignedFree(void *ptr)
418 {
419 auto aligned = reinterpret_cast<uint8_t *>(ptr);
420 auto offset = aligned[-1];
421 auto allocation = aligned - offset;
422 delete[] allocation;
423 }
424
425 template<typename T>
atomicLoad(void * ptr,void * ret,llvm::AtomicOrdering ordering)426 static void atomicLoad(void *ptr, void *ret, llvm::AtomicOrdering ordering)
427 {
428 *reinterpret_cast<T *>(ret) = std::atomic_load_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), rr::atomicOrdering(ordering));
429 }
430
431 template<typename T>
atomicStore(void * ptr,void * val,llvm::AtomicOrdering ordering)432 static void atomicStore(void *ptr, void *val, llvm::AtomicOrdering ordering)
433 {
434 std::atomic_store_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), *reinterpret_cast<T *>(val), rr::atomicOrdering(ordering));
435 }
436
437 #ifdef __ANDROID__
438 template<typename F>
sync_fetch_and_op(uint32_t volatile * ptr,uint32_t val,F f)439 static uint32_t sync_fetch_and_op(uint32_t volatile *ptr, uint32_t val, F f)
440 {
441 // Build an arbitrary op out of looped CAS
442 for(;;)
443 {
444 uint32_t expected = *ptr;
445 uint32_t desired = f(expected, val);
446
447 if(expected == __sync_val_compare_and_swap_4(ptr, expected, desired))
448 {
449 return expected;
450 }
451 }
452 }
453 #endif
454
455 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
456 class ExternalSymbolGenerator : public llvm::orc::DefinitionGenerator
457 #else
458 class ExternalSymbolGenerator : public llvm::orc::JITDylib::DefinitionGenerator
459 #endif
460 {
461 struct Atomic
462 {
load__anonc2fb15830111::ExternalSymbolGenerator::Atomic463 static void load(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering)
464 {
465 switch(size)
466 {
467 case 1: atomicLoad<uint8_t>(ptr, ret, ordering); break;
468 case 2: atomicLoad<uint16_t>(ptr, ret, ordering); break;
469 case 4: atomicLoad<uint32_t>(ptr, ret, ordering); break;
470 case 8: atomicLoad<uint64_t>(ptr, ret, ordering); break;
471 default:
472 UNIMPLEMENTED_NO_BUG("Atomic::load(size: %d)", int(size));
473 }
474 }
store__anonc2fb15830111::ExternalSymbolGenerator::Atomic475 static void store(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering)
476 {
477 switch(size)
478 {
479 case 1: atomicStore<uint8_t>(ptr, ret, ordering); break;
480 case 2: atomicStore<uint16_t>(ptr, ret, ordering); break;
481 case 4: atomicStore<uint32_t>(ptr, ret, ordering); break;
482 case 8: atomicStore<uint64_t>(ptr, ret, ordering); break;
483 default:
484 UNIMPLEMENTED_NO_BUG("Atomic::store(size: %d)", int(size));
485 }
486 }
487 };
488
nop()489 static void nop() {}
neverCalled()490 static void neverCalled() { UNREACHABLE("Should never be called"); }
491
coroutine_alloc_frame(size_t size)492 static void *coroutine_alloc_frame(size_t size) { return alignedAlloc(size, 16); }
coroutine_free_frame(void * ptr)493 static void coroutine_free_frame(void *ptr) { alignedFree(ptr); }
494
495 #ifdef __ANDROID__
496 // forwarders since we can't take address of builtins
sync_synchronize()497 static void sync_synchronize() { __sync_synchronize(); }
sync_fetch_and_add_4(uint32_t * ptr,uint32_t val)498 static uint32_t sync_fetch_and_add_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_add_4(ptr, val); }
sync_fetch_and_and_4(uint32_t * ptr,uint32_t val)499 static uint32_t sync_fetch_and_and_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_and_4(ptr, val); }
sync_fetch_and_or_4(uint32_t * ptr,uint32_t val)500 static uint32_t sync_fetch_and_or_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_or_4(ptr, val); }
sync_fetch_and_xor_4(uint32_t * ptr,uint32_t val)501 static uint32_t sync_fetch_and_xor_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_xor_4(ptr, val); }
sync_fetch_and_sub_4(uint32_t * ptr,uint32_t val)502 static uint32_t sync_fetch_and_sub_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_sub_4(ptr, val); }
sync_lock_test_and_set_4(uint32_t * ptr,uint32_t val)503 static uint32_t sync_lock_test_and_set_4(uint32_t *ptr, uint32_t val) { return __sync_lock_test_and_set_4(ptr, val); }
sync_val_compare_and_swap_4(uint32_t * ptr,uint32_t expected,uint32_t desired)504 static uint32_t sync_val_compare_and_swap_4(uint32_t *ptr, uint32_t expected, uint32_t desired) { return __sync_val_compare_and_swap_4(ptr, expected, desired); }
505
sync_fetch_and_max_4(uint32_t * ptr,uint32_t val)506 static uint32_t sync_fetch_and_max_4(uint32_t *ptr, uint32_t val)
507 {
508 return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::max(a, b); });
509 }
sync_fetch_and_min_4(uint32_t * ptr,uint32_t val)510 static uint32_t sync_fetch_and_min_4(uint32_t *ptr, uint32_t val)
511 {
512 return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::min(a, b); });
513 }
sync_fetch_and_umax_4(uint32_t * ptr,uint32_t val)514 static uint32_t sync_fetch_and_umax_4(uint32_t *ptr, uint32_t val)
515 {
516 return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::max(a, b); });
517 }
sync_fetch_and_umin_4(uint32_t * ptr,uint32_t val)518 static uint32_t sync_fetch_and_umin_4(uint32_t *ptr, uint32_t val)
519 {
520 return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::min(a, b); });
521 }
522 #endif
523
524 class Resolver
525 {
526 public:
527 using FunctionMap = llvm::StringMap<void *>;
528
529 FunctionMap functions;
530
Resolver()531 Resolver()
532 {
533 #ifdef ENABLE_RR_PRINT
534 functions.try_emplace("rr::DebugPrintf", reinterpret_cast<void *>(rr::DebugPrintf));
535 #endif
536 functions.try_emplace("nop", reinterpret_cast<void *>(nop));
537 functions.try_emplace("floorf", reinterpret_cast<void *>(floorf));
538 functions.try_emplace("nearbyintf", reinterpret_cast<void *>(nearbyintf));
539 functions.try_emplace("truncf", reinterpret_cast<void *>(truncf));
540 functions.try_emplace("printf", reinterpret_cast<void *>(printf));
541 functions.try_emplace("puts", reinterpret_cast<void *>(puts));
542 functions.try_emplace("fmodf", reinterpret_cast<void *>(fmodf));
543
544 functions.try_emplace("sinf", reinterpret_cast<void *>(sinf));
545 functions.try_emplace("cosf", reinterpret_cast<void *>(cosf));
546 functions.try_emplace("asinf", reinterpret_cast<void *>(asinf));
547 functions.try_emplace("acosf", reinterpret_cast<void *>(acosf));
548 functions.try_emplace("atanf", reinterpret_cast<void *>(atanf));
549 functions.try_emplace("sinhf", reinterpret_cast<void *>(sinhf));
550 functions.try_emplace("coshf", reinterpret_cast<void *>(coshf));
551 functions.try_emplace("tanhf", reinterpret_cast<void *>(tanhf));
552 functions.try_emplace("asinhf", reinterpret_cast<void *>(asinhf));
553 functions.try_emplace("acoshf", reinterpret_cast<void *>(acoshf));
554 functions.try_emplace("atanhf", reinterpret_cast<void *>(atanhf));
555 functions.try_emplace("atan2f", reinterpret_cast<void *>(atan2f));
556 functions.try_emplace("powf", reinterpret_cast<void *>(powf));
557 functions.try_emplace("expf", reinterpret_cast<void *>(expf));
558 functions.try_emplace("logf", reinterpret_cast<void *>(logf));
559 functions.try_emplace("exp2f", reinterpret_cast<void *>(exp2f));
560 functions.try_emplace("log2f", reinterpret_cast<void *>(log2f));
561 functions.try_emplace("fmaf", reinterpret_cast<void *>(fmaf));
562
563 functions.try_emplace("fmod", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(fmod)));
564 functions.try_emplace("sin", reinterpret_cast<void *>(static_cast<double (*)(double)>(sin)));
565 functions.try_emplace("cos", reinterpret_cast<void *>(static_cast<double (*)(double)>(cos)));
566 functions.try_emplace("asin", reinterpret_cast<void *>(static_cast<double (*)(double)>(asin)));
567 functions.try_emplace("acos", reinterpret_cast<void *>(static_cast<double (*)(double)>(acos)));
568 functions.try_emplace("atan", reinterpret_cast<void *>(static_cast<double (*)(double)>(atan)));
569 functions.try_emplace("sinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(sinh)));
570 functions.try_emplace("cosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(cosh)));
571 functions.try_emplace("tanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(tanh)));
572 functions.try_emplace("asinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(asinh)));
573 functions.try_emplace("acosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(acosh)));
574 functions.try_emplace("atanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(atanh)));
575 functions.try_emplace("atan2", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(atan2)));
576 functions.try_emplace("pow", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(pow)));
577 functions.try_emplace("exp", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp)));
578 functions.try_emplace("log", reinterpret_cast<void *>(static_cast<double (*)(double)>(log)));
579 functions.try_emplace("exp2", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp2)));
580 functions.try_emplace("log2", reinterpret_cast<void *>(static_cast<double (*)(double)>(log2)));
581
582 functions.try_emplace("atomic_load", reinterpret_cast<void *>(Atomic::load));
583 functions.try_emplace("atomic_store", reinterpret_cast<void *>(Atomic::store));
584
585 // FIXME(b/119409619): use an allocator here so we can control all memory allocations
586 functions.try_emplace("coroutine_alloc_frame", reinterpret_cast<void *>(coroutine_alloc_frame));
587 functions.try_emplace("coroutine_free_frame", reinterpret_cast<void *>(coroutine_free_frame));
588
589 functions.try_emplace("memset", reinterpret_cast<void *>(memset));
590
591 #ifdef __APPLE__
592 functions.try_emplace("__sincosf_stret", reinterpret_cast<void *>(__sincosf_stret));
593 #elif defined(__linux__)
594 functions.try_emplace("sincosf", reinterpret_cast<void *>(sincosf));
595 #elif defined(_WIN64)
596 functions.try_emplace("__chkstk", reinterpret_cast<void *>(__chkstk));
597 #elif defined(_WIN32)
598 functions.try_emplace("_chkstk", reinterpret_cast<void *>(_chkstk));
599 #endif
600
601 #ifdef __ARM_EABI__
602 functions.try_emplace("__aeabi_idivmod", reinterpret_cast<void *>(__aeabi_idivmod));
603 #endif
604 #ifdef __ANDROID__
605 functions.try_emplace("aeabi_unwind_cpp_pr0", reinterpret_cast<void *>(neverCalled));
606 functions.try_emplace("sync_synchronize", reinterpret_cast<void *>(sync_synchronize));
607 functions.try_emplace("sync_fetch_and_add_4", reinterpret_cast<void *>(sync_fetch_and_add_4));
608 functions.try_emplace("sync_fetch_and_and_4", reinterpret_cast<void *>(sync_fetch_and_and_4));
609 functions.try_emplace("sync_fetch_and_or_4", reinterpret_cast<void *>(sync_fetch_and_or_4));
610 functions.try_emplace("sync_fetch_and_xor_4", reinterpret_cast<void *>(sync_fetch_and_xor_4));
611 functions.try_emplace("sync_fetch_and_sub_4", reinterpret_cast<void *>(sync_fetch_and_sub_4));
612 functions.try_emplace("sync_lock_test_and_set_4", reinterpret_cast<void *>(sync_lock_test_and_set_4));
613 functions.try_emplace("sync_val_compare_and_swap_4", reinterpret_cast<void *>(sync_val_compare_and_swap_4));
614 functions.try_emplace("sync_fetch_and_max_4", reinterpret_cast<void *>(sync_fetch_and_max_4));
615 functions.try_emplace("sync_fetch_and_min_4", reinterpret_cast<void *>(sync_fetch_and_min_4));
616 functions.try_emplace("sync_fetch_and_umax_4", reinterpret_cast<void *>(sync_fetch_and_umax_4));
617 functions.try_emplace("sync_fetch_and_umin_4", reinterpret_cast<void *>(sync_fetch_and_umin_4));
618
619 # if defined(__i386__)
620 // TODO(b/172974501): Workaround for an x86-32 issue where an R_386_PC32 relocation is used
621 // When calling a C function from Reactor code, who's address is not associated with any symbol
622 // (since it's an absolute constant), but it still invokes the symbol resolver for "".
623 functions.try_emplace("", nullptr);
624 # endif
625 #endif
626 #if __has_feature(memory_sanitizer)
627 functions.try_emplace("__emutls_get_address", reinterpret_cast<void *>(rr::getTLSAddress));
628 functions.try_emplace("__emutls_v.__msan_param_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::param)));
629 functions.try_emplace("__emutls_v.__msan_param_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::param_origin)));
630 functions.try_emplace("__emutls_v.__msan_retval_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::retval)));
631 functions.try_emplace("__emutls_v.__msan_retval_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::retval_origin)));
632 functions.try_emplace("__emutls_v.__msan_va_arg_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg)));
633 functions.try_emplace("__emutls_v.__msan_va_arg_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg_origin)));
634 functions.try_emplace("__emutls_v.__msan_va_arg_overflow_size_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg_overflow_size)));
635 functions.try_emplace("__emutls_v.__msan_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::origin)));
636
637 functions.try_emplace("__msan_unpoison", reinterpret_cast<void *>(__msan_unpoison));
638 functions.try_emplace("__msan_unpoison_param", reinterpret_cast<void *>(__msan_unpoison_param));
639 #endif
640 }
641 };
642
tryToGenerate(llvm::orc::LookupState & state,llvm::orc::LookupKind kind,llvm::orc::JITDylib & dylib,llvm::orc::JITDylibLookupFlags flags,const llvm::orc::SymbolLookupSet & set)643 llvm::Error tryToGenerate(
644 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
645 llvm::orc::LookupState &state,
646 #endif
647 llvm::orc::LookupKind kind,
648 llvm::orc::JITDylib &dylib,
649 llvm::orc::JITDylibLookupFlags flags,
650 const llvm::orc::SymbolLookupSet &set) override
651 {
652 static Resolver resolver;
653
654 llvm::orc::SymbolMap symbols;
655
656 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
657 std::string missing;
658 #endif // !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
659
660 for(auto symbol : set)
661 {
662 auto name = symbol.first;
663
664 #if defined(__APPLE__)
665 // Trim the underscore from the start of the symbol. LLVM adds it for Mach-O mangling convention.
666 ASSERT((*name)[0] == '_');
667 auto unmangled = (*name).drop_front(1);
668 #else
669 auto unmangled = *name;
670 #endif
671
672 #if LLVM_VERSION_MAJOR < 17
673 auto toSymbol = [](void *ptr) {
674 return llvm::JITEvaluatedSymbol(
675 static_cast<llvm::JITTargetAddress>(reinterpret_cast<uintptr_t>(ptr)),
676 llvm::JITSymbolFlags::Exported);
677 };
678 #else
679 auto toSymbol = [](void *ptr) {
680 return llvm::orc::ExecutorSymbolDef{
681 llvm::orc::ExecutorAddr(reinterpret_cast<uintptr_t>(ptr)),
682 llvm::JITSymbolFlags::Exported,
683 };
684 };
685 #endif
686
687 auto it = resolver.functions.find(unmangled.str());
688 if(it != resolver.functions.end())
689 {
690 symbols[name] = toSymbol(it->second);
691 continue;
692 }
693
694 #if __has_feature(memory_sanitizer) || (__has_feature(address_sanitizer) && ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED)
695 // Sanitizers use a dynamically linked runtime. Instrumented routines reference some
696 // symbols from this library. Look them up dynamically in the default namespace.
697 // Note this approach should not be used for other symbols, since they might not be
698 // visible (e.g. due to static linking), we may wish to provide an alternate
699 // implementation, and/or it would be a security vulnerability.
700
701 void *address = dlsym(RTLD_DEFAULT, unmangled.data());
702
703 if(address)
704 {
705 symbols[name] = toSymbol(address);
706 continue;
707 }
708 #endif
709
710 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
711 missing += (missing.empty() ? "'" : ", '") + unmangled.str() + "'";
712 #endif
713 }
714
715 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
716 // Missing functions will likely make the module fail in non-obvious ways.
717 if(!missing.empty())
718 {
719 WARN("Missing external functions: %s", missing.c_str());
720 }
721 #endif
722
723 if(symbols.empty())
724 {
725 return llvm::Error::success();
726 }
727
728 return dylib.define(llvm::orc::absoluteSymbols(std::move(symbols)));
729 }
730 };
731
732 // As we must support different LLVM versions, add a generic Unwrap for functions that return Expected<T> or the actual T.
733 // TODO(b/165000222): Remove after LLVM 11 upgrade
734 template<typename T>
Unwrap(llvm::Expected<T> && v)735 T &Unwrap(llvm::Expected<T> &&v)
736 {
737 assert(v);
738 return v.get();
739 }
740 template<typename T>
Unwrap(T && v)741 T &Unwrap(T &&v)
742 {
743 return v;
744 }
745
746 // Sets *fatal to true if a diagnostic is received which makes a routine invalid or unusable.
747 struct FatalDiagnosticsHandler : public llvm::DiagnosticHandler
748 {
FatalDiagnosticsHandler__anonc2fb15830111::FatalDiagnosticsHandler749 FatalDiagnosticsHandler(bool *fatal)
750 : fatal(fatal)
751 {}
752
handleDiagnostics__anonc2fb15830111::FatalDiagnosticsHandler753 bool handleDiagnostics(const llvm::DiagnosticInfo &info) override
754 {
755 switch(info.getSeverity())
756 {
757 case llvm::DS_Error:
758 ASSERT_MSG(false, "LLVM JIT compilation failure");
759 *fatal = true;
760 break;
761 case llvm::DS_Warning:
762 if(info.getKind() == llvm::DK_StackSize)
763 {
764 // Stack size limit exceeded
765 *fatal = true;
766 }
767 break;
768 case llvm::DS_Remark:
769 break;
770 case llvm::DS_Note:
771 break;
772 }
773
774 return true; // Diagnostic handled, don't let LLVM print it.
775 }
776
777 bool *fatal;
778 };
779
780 // JITRoutine is a rr::Routine that holds a LLVM JIT session, compiler and
781 // object layer as each routine may require different target machine
782 // settings and no Reactor routine directly links against another.
783 class JITRoutine : public rr::Routine
784 {
785 public:
JITRoutine(std::unique_ptr<llvm::Module> module,std::unique_ptr<llvm::LLVMContext> context,const char * name,llvm::Function ** funcs,size_t count)786 JITRoutine(
787 std::unique_ptr<llvm::Module> module,
788 std::unique_ptr<llvm::LLVMContext> context,
789 const char *name,
790 llvm::Function **funcs,
791 size_t count)
792 : name(name)
793 #if LLVM_VERSION_MAJOR >= 13
794 , session(std::move(Unwrap(llvm::orc::SelfExecutorProcessControl::Create())))
795 #endif
796 #if USE_LEGACY_OBJECT_LINKING_LAYER
797 , objectLayer(session, [this]() {
798 return std::make_unique<llvm::SectionMemoryManager>(&memoryMapper);
799 })
800 #else
801 , objectLayer(session, llvm::cantFail(llvm::jitlink::InProcessMemoryManager::Create()))
802 #endif
803 , addresses(count)
804 {
805 bool fatalCompileIssue = false;
806 context->setDiagnosticHandler(std::make_unique<FatalDiagnosticsHandler>(&fatalCompileIssue), true);
807
808 #ifdef ENABLE_RR_DEBUG_INFO
809 // TODO(b/165000222): Update this on next LLVM roll.
810 // https://github.com/llvm/llvm-project/commit/98f2bb4461072347dcca7d2b1b9571b3a6525801
811 // introduces RTDyldObjectLinkingLayer::registerJITEventListener().
812 // The current API does not appear to have any way to bind the
813 // rr::DebugInfo::NotifyFreeingObject event.
814 # if LLVM_VERSION_MAJOR >= 12
815 objectLayer.setNotifyLoaded([](llvm::orc::MaterializationResponsibility &R,
816 const llvm::object::ObjectFile &obj,
__anonc2fb15830b02(llvm::orc::MaterializationResponsibility &R, const llvm::object::ObjectFile &obj, const llvm::RuntimeDyld::LoadedObjectInfo &l) 817 const llvm::RuntimeDyld::LoadedObjectInfo &l) {
818 static std::atomic<uint64_t> unique_key{ 0 };
819 rr::DebugInfo::NotifyObjectEmitted(unique_key++, obj, l);
820 });
821 # else
822 objectLayer.setNotifyLoaded([](llvm::orc::VModuleKey,
823 const llvm::object::ObjectFile &obj,
__anonc2fb15830c02(llvm::orc::VModuleKey, const llvm::object::ObjectFile &obj, const llvm::RuntimeDyld::LoadedObjectInfo &l) 824 const llvm::RuntimeDyld::LoadedObjectInfo &l) {
825 static std::atomic<uint64_t> unique_key{ 0 };
826 rr::DebugInfo::NotifyObjectEmitted(unique_key++, obj, l);
827 });
828 # endif
829 #endif // ENABLE_RR_DEBUG_INFO
830
831 if(JITGlobals::get()->getTargetTriple().isOSBinFormatCOFF())
832 {
833 // Hack to support symbol visibility in COFF.
834 // Matches hack in llvm::orc::LLJIT::createObjectLinkingLayer().
835 // See documentation on these functions for more detail.
836 objectLayer.setOverrideObjectFlagsWithResponsibilityFlags(true);
837 objectLayer.setAutoClaimResponsibilityForObjectSymbols(true);
838 }
839
840 llvm::SmallVector<llvm::orc::SymbolStringPtr, 8> functionNames(count);
841 llvm::orc::MangleAndInterner mangle(session, JITGlobals::get()->getDataLayout());
842
843 for(size_t i = 0; i < count; i++)
844 {
845 llvm::Function *func = funcs[i];
846
847 if(!func->hasName())
848 {
849 func->setName("f" + llvm::Twine(i).str());
850 }
851
852 functionNames[i] = mangle(func->getName());
853 }
854
855 #ifdef ENABLE_RR_EMIT_ASM_FILE
856 const auto asmFilename = rr::AsmFile::generateFilename(REACTOR_ASM_EMIT_DIR, name);
857 rr::AsmFile::emitAsmFile(asmFilename, JITGlobals::get()->getTargetMachineBuilder(), *module);
858 #endif
859
860 // Once the module is passed to the compileLayer, the llvm::Functions are freed.
861 // Make sure funcs are not referenced after this point.
862 funcs = nullptr;
863
864 llvm::orc::IRCompileLayer compileLayer(session, objectLayer, std::make_unique<llvm::orc::ConcurrentIRCompiler>(JITGlobals::get()->getTargetMachineBuilder()));
865 llvm::orc::JITDylib &dylib(Unwrap(session.createJITDylib("<routine>")));
866 dylib.addGenerator(std::make_unique<ExternalSymbolGenerator>());
867
868 llvm::cantFail(compileLayer.add(dylib, llvm::orc::ThreadSafeModule(std::move(module), std::move(context))));
869
870 // Resolve the function addresses.
871 for(size_t i = 0; i < count; i++)
872 {
873 fatalCompileIssue = false; // May be set to true by session.lookup()
874
875 // This is where the actual compilation happens.
876 auto symbol = session.lookup({ &dylib }, functionNames[i]);
877
878 ASSERT_MSG(symbol, "Failed to lookup address of routine function %d: %s",
879 (int)i, llvm::toString(symbol.takeError()).c_str());
880
881 if(fatalCompileIssue)
882 {
883 addresses[i] = nullptr;
884 }
885 else // Successful compilation
886 {
887 #if LLVM_VERSION_MAJOR < 17
888 addresses[i] = reinterpret_cast<void *>(static_cast<intptr_t>(symbol->getAddress()));
889 #else
890 addresses[i] = reinterpret_cast<void *>(static_cast<intptr_t>(symbol->getAddress().getValue()));
891 #endif
892 }
893 }
894
895 #ifdef ENABLE_RR_EMIT_ASM_FILE
896 rr::AsmFile::fixupAsmFile(asmFilename, addresses);
897 #endif
898 }
899
~JITRoutine()900 ~JITRoutine()
901 {
902 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
903 if(auto err = session.endSession())
904 {
905 session.reportError(std::move(err));
906 }
907 #endif
908 }
909
getEntry(int index) const910 const void *getEntry(int index) const override
911 {
912 return addresses[index];
913 }
914
915 private:
916 std::string name;
917 llvm::orc::ExecutionSession session;
918 MemoryMapper memoryMapper;
919 #if USE_LEGACY_OBJECT_LINKING_LAYER
920 llvm::orc::RTDyldObjectLinkingLayer objectLayer;
921 #else
922 llvm::orc::ObjectLinkingLayer objectLayer;
923 #endif
924 std::vector<const void *> addresses;
925 };
926
927 } // anonymous namespace
928
929 namespace rr {
930
JITBuilder()931 JITBuilder::JITBuilder()
932 : context(new llvm::LLVMContext())
933 , module(new llvm::Module("", *context))
934 , builder(new llvm::IRBuilder<>(*context))
935 {
936 module->setTargetTriple(LLVM_DEFAULT_TARGET_TRIPLE);
937 module->setDataLayout(JITGlobals::get()->getDataLayout());
938
939 msanInstrumentation = getPragmaState(MemorySanitizerInstrumentation);
940 }
941
runPasses()942 void JITBuilder::runPasses()
943 {
944 #if defined(ENABLE_RR_LLVM_IR_VERIFICATION) || !defined(NDEBUG)
945 if(llvm::verifyModule(*module, &llvm::errs()))
946 {
947 llvm::report_fatal_error("Invalid LLVM module");
948 }
949 #endif
950
951 int optimizationLevel = getPragmaState(OptimizationLevel);
952
953 #ifdef ENABLE_RR_DEBUG_INFO
954 if(debugInfo != nullptr)
955 {
956 optimizationLevel = 0; // Don't optimize if we're generating debug info.
957 }
958 #endif // ENABLE_RR_DEBUG_INFO
959
960 #if LLVM_VERSION_MAJOR >= 13 // New pass manager
961 llvm::LoopAnalysisManager lam;
962 llvm::FunctionAnalysisManager fam;
963 llvm::CGSCCAnalysisManager cgam;
964 llvm::ModuleAnalysisManager mam;
965 llvm::PassBuilder pb;
966
967 pb.registerModuleAnalyses(mam);
968 pb.registerCGSCCAnalyses(cgam);
969 pb.registerFunctionAnalyses(fam);
970 pb.registerLoopAnalyses(lam);
971 pb.crossRegisterProxies(lam, fam, cgam, mam);
972
973 llvm::ModulePassManager pm;
974 llvm::FunctionPassManager fpm;
975
976 if(coroutine.id)
977 {
978 // Adds mandatory coroutine transforms.
979 pm = pb.buildO0DefaultPipeline(llvm::OptimizationLevel::O0);
980 }
981
982 if(optimizationLevel > 0)
983 {
984 fpm.addPass(llvm::SROAPass(llvm::SROAOptions::PreserveCFG));
985 fpm.addPass(llvm::InstCombinePass());
986 }
987
988 if(!fpm.isEmpty())
989 {
990 pm.addPass(llvm::createModuleToFunctionPassAdaptor(std::move(fpm)));
991 }
992
993 if(__has_feature(memory_sanitizer) && msanInstrumentation)
994 {
995 llvm::MemorySanitizerOptions msanOpts(0 /* TrackOrigins */, false /* Recover */, false /* Kernel */, true /* EagerChecks */);
996 pm.addPass(llvm::MemorySanitizerPass(msanOpts));
997 }
998
999 if(__has_feature(address_sanitizer) && ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED)
1000 {
1001 pm.addPass(llvm::AddressSanitizerPass(llvm::AddressSanitizerOptions{}));
1002 }
1003
1004 pm.run(*module, mam);
1005 #else // Legacy pass manager
1006 llvm::legacy::PassManager passManager;
1007
1008 if(coroutine.id)
1009 {
1010 // Run mandatory coroutine transforms.
1011 passManager.add(llvm::createCoroEarlyLegacyPass());
1012 passManager.add(llvm::createCoroSplitLegacyPass());
1013 passManager.add(llvm::createCoroElideLegacyPass());
1014 passManager.add(llvm::createBarrierNoopPass());
1015 passManager.add(llvm::createCoroCleanupLegacyPass());
1016 }
1017
1018 if(optimizationLevel > 0)
1019 {
1020 passManager.add(llvm::createSROAPass());
1021 passManager.add(llvm::createInstructionCombiningPass());
1022 }
1023
1024 if(__has_feature(memory_sanitizer) && msanInstrumentation)
1025 {
1026 llvm::MemorySanitizerOptions msanOpts(0 /* TrackOrigins */, false /* Recover */, false /* Kernel */);
1027 passManager.add(llvm::createMemorySanitizerLegacyPassPass(msanOpts));
1028 }
1029
1030 if(__has_feature(address_sanitizer) && ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED)
1031 {
1032 passManager.add(llvm::createAddressSanitizerFunctionPass());
1033 }
1034
1035 passManager.run(*module);
1036 #endif
1037 }
1038
acquireRoutine(const char * name,llvm::Function ** funcs,size_t count)1039 std::shared_ptr<rr::Routine> JITBuilder::acquireRoutine(const char *name, llvm::Function **funcs, size_t count)
1040 {
1041 ASSERT(module);
1042 return std::make_shared<JITRoutine>(std::move(module), std::move(context), name, funcs, count);
1043 }
1044
1045 } // namespace rr
1046