1 /*
2 * Copyright (c) 2022 Alex Fan <[email protected]>
3 * SPDX-License-Identifier: MIT
4 */
5
6 #include "util/detect.h"
7 #include "util/u_cpu_detect.h"
8 #include "util/u_debug.h"
9 #include "util/os_time.h"
10 #include <string>
11 #include <vector>
12 #include <mutex>
13 #include <cstdlib>
14 #include "lp_bld.h"
15 #include "lp_bld_debug.h"
16 #include "lp_bld_init.h"
17 #include "lp_bld_coro.h"
18 #include "lp_bld_misc.h"
19 #include "lp_bld_printf.h"
20 #include "lp_bld_passmgr.h"
21 #include "lp_bld_type.h"
22
23 #include <llvm/Config/llvm-config.h>
24 #include <llvm-c/Core.h>
25 #include <llvm-c/Orc.h>
26 #include <llvm-c/LLJIT.h>
27 #include <llvm-c/TargetMachine.h>
28 #include <llvm-c/Support.h>
29
30 #include <llvm-c/Analysis.h>
31 #if LLVM_VERSION_MAJOR < 17
32 #include <llvm-c/Transforms/Scalar.h>
33 #if LLVM_VERSION_MAJOR >= 7
34 #include <llvm-c/Transforms/Utils.h>
35 #endif
36 #endif
37 #include <llvm-c/BitWriter.h>
38
39 #include <llvm/ADT/StringMap.h>
40 #include <llvm/ExecutionEngine/Orc/LLJIT.h>
41 #include <llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h>
42 #include <llvm/ExecutionEngine/Orc/CompileUtils.h>
43 #include <llvm/ExecutionEngine/ObjectCache.h>
44 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
45 #include <llvm/Target/TargetMachine.h>
46 #include <llvm/Support/TargetSelect.h>
47 #include <llvm/Support/Casting.h>
48 #if LLVM_VERSION_MAJOR >= 18
49 #include <llvm/TargetParser/Host.h>
50 #else
51 #include <llvm/Support/Host.h>
52 #endif
53 #include <llvm/Support/CBindingWrapping.h>
54 #if LLVM_USE_INTEL_JITEVENTS
55 #include <llvm/ExecutionEngine/JITEventListener.h>
56 #endif
57
58 /* conflict with ObjectLinkingLayer.h */
59 #include "util/u_memory.h"
60
61 #if DETECT_ARCH_RISCV64 == 1 || DETECT_ARCH_RISCV32 == 1 || DETECT_ARCH_LOONGARCH64 == 1 || (defined(_WIN32) && LLVM_VERSION_MAJOR >= 15)
62 /* use ObjectLinkingLayer (JITLINK backend) */
63 #define USE_JITLINK
64 #endif
65 /* else use old RTDyldObjectLinkingLayer (RuntimeDyld backend) */
66
67 namespace {
68
69 class LPObjectCacheORC : public llvm::ObjectCache {
70 private:
71 bool has_object;
72 std::string mid;
73 struct lp_cached_code *cache_out;
74 public:
LPObjectCacheORC(struct lp_cached_code * cache)75 LPObjectCacheORC(struct lp_cached_code *cache) {
76 cache_out = cache;
77 has_object = false;
78 }
79
~LPObjectCacheORC()80 ~LPObjectCacheORC() {
81 }
notifyObjectCompiled(const llvm::Module * M,llvm::MemoryBufferRef Obj)82 void notifyObjectCompiled(const llvm::Module *M, llvm::MemoryBufferRef Obj) override {
83 const std::string ModuleID = M->getModuleIdentifier();
84 if (has_object)
85 fprintf(stderr, "CACHE ALREADY HAS MODULE OBJECT\n");
86 if (mid == ModuleID)
87 fprintf(stderr, "CACHING ANOTHER MODULE\n");
88 has_object = true;
89 mid = ModuleID;
90 cache_out->data_size = Obj.getBufferSize();
91 cache_out->data = malloc(cache_out->data_size);
92 memcpy(cache_out->data, Obj.getBufferStart(), cache_out->data_size);
93 }
94
getObject(const llvm::Module * M)95 std::unique_ptr<llvm::MemoryBuffer> getObject(const llvm::Module *M) override {
96 const std::string ModuleID = M->getModuleIdentifier();
97 if (cache_out->data_size)
98 return llvm::MemoryBuffer::getMemBuffer(llvm::StringRef((const char *)cache_out->data, cache_out->data_size), "", false);
99 return NULL;
100 }
101
102 };
103
104 class LPJit;
105
106 void lpjit_exit();
107
108 class LLVMEnsureMultithreaded {
109 public:
LLVMEnsureMultithreaded()110 LLVMEnsureMultithreaded()
111 {
112 if (!LLVMIsMultithreaded()) {
113 LLVMStartMultithreaded();
114 }
115 }
116 };
117
118 LLVMEnsureMultithreaded lLVMEnsureMultithreaded;
119
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::ThreadSafeContext,LLVMOrcThreadSafeContextRef)120 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::ThreadSafeContext,
121 LLVMOrcThreadSafeContextRef)
122 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::IRTransformLayer,
123 LLVMOrcIRTransformLayerRef)
124 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITDylib, LLVMOrcJITDylibRef)
125 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITTargetMachineBuilder,
126 LLVMOrcJITTargetMachineBuilderRef)
127 LLVMTargetMachineRef wrap(const llvm::TargetMachine *P) {
128 return reinterpret_cast<LLVMTargetMachineRef>(const_cast<llvm::TargetMachine*>(P));
129 }
130
131 llvm::ExitOnError ExitOnErr;
132
get_module_name(LLVMModuleRef mod)133 inline const char* get_module_name(LLVMModuleRef mod) {
134 using llvm::Module;
135 return llvm::unwrap(mod)->getModuleIdentifier().c_str();
136 }
137
138 once_flag init_lpjit_once_flag = ONCE_FLAG_INIT;
139
140 /* A JIT singleton built upon LLJIT */
141 class LPJit
142 {
143 public:
get_instance()144 static LPJit* get_instance() {
145 call_once(&init_lpjit_once_flag, init_lpjit);
146 return jit;
147 }
148
find_gallivm_state(LLVMModuleRef mod)149 gallivm_state *find_gallivm_state(LLVMModuleRef mod) {
150 #if DEBUG
151 using llvm::Module;
152 auto I = gallivm_modules.find(llvm::unwrap(mod)->getModuleIdentifier());
153 if (I == gallivm_modules.end()) {
154 debug_printf("No gallivm state found for module: %s", get_module_name(mod));
155 return NULL;
156 }
157 return I->second;
158 #endif
159 return NULL;
160 }
161
get_unique_name(const char * name)162 static char *get_unique_name(const char* name) {
163 LPJit* jit = get_instance();
164 size_t size = name == NULL? 16: strlen(name) + 16;
165 char *name_uniq = (char *)MALLOC(size);
166 if (!name_uniq) {
167 return NULL;
168 }
169 do {
170 snprintf(name_uniq, size, "%s_%u", name, jit->jit_dylib_count++);
171 } while(jit->lljit->getExecutionSession().getJITDylibByName(name_uniq));
172 return name_uniq;
173 }
174
create_jit_dylib(const char * name)175 static LLVMOrcJITDylibRef create_jit_dylib(const char * name) {
176 using llvm::orc::JITDylib;
177 LPJit* jit = get_instance();
178 JITDylib& tmp = ExitOnErr(jit->lljit->createJITDylib(name));
179 return wrap(&tmp);
180 }
181
register_gallivm_state(gallivm_state * gallivm)182 static void register_gallivm_state(gallivm_state *gallivm) {
183 #if DEBUG
184 LPJit* jit = get_instance();
185 jit->gallivm_modules[gallivm->module_name] = gallivm;
186 #endif
187 }
188
deregister_gallivm_state(gallivm_state * gallivm)189 static void deregister_gallivm_state(gallivm_state *gallivm) {
190 #if DEBUG
191 LPJit* jit = get_instance();
192 (void)jit->gallivm_modules.erase(gallivm->module_name);
193 #endif
194 }
195
add_ir_module_to_jd(LLVMOrcThreadSafeContextRef ts_context,LLVMModuleRef mod,LLVMOrcJITDylibRef jd)196 static void add_ir_module_to_jd(
197 LLVMOrcThreadSafeContextRef ts_context,
198 LLVMModuleRef mod,
199 LLVMOrcJITDylibRef jd) {
200 using llvm::Module;
201 using llvm::orc::ThreadSafeModule;
202 using llvm::orc::JITDylib;
203 ThreadSafeModule tsm(
204 std::unique_ptr<Module>(llvm::unwrap(mod)), *::unwrap(ts_context));
205 ExitOnErr(get_instance()->lljit->addIRModule(
206 *::unwrap(jd), std::move(tsm)
207 ));
208 }
209
add_mapping_to_jd(LLVMValueRef sym,void * addr,LLVMOrcJITDylibRef jd)210 static void add_mapping_to_jd(
211 LLVMValueRef sym,
212 void *addr,
213 LLVMOrcJITDylibRef jd) {
214 #if LLVM_VERSION_MAJOR >= 17
215 using llvm::orc::ExecutorAddr;
216 using llvm::orc::ExecutorSymbolDef;
217 using llvm::JITSymbolFlags;
218 #else
219 using llvm::JITEvaluatedSymbol;
220 #endif
221 using llvm::orc::ExecutionSession;
222 using llvm::orc::JITDylib;
223 using llvm::orc::SymbolMap;
224 JITDylib* JD = ::unwrap(jd);
225 auto& es = LPJit::get_instance()->lljit->getExecutionSession();
226 auto name = es.intern(llvm::unwrap(sym)->getName());
227 SymbolMap map(1);
228 #if LLVM_VERSION_MAJOR >= 17
229 map[name] = ExecutorSymbolDef(ExecutorAddr::fromPtr(addr), JITSymbolFlags::Exported);
230 #else
231 map[name] = JITEvaluatedSymbol::fromPointer(addr);
232 #endif
233 auto munit = llvm::orc::absoluteSymbols(map);
234 llvm::cantFail(JD->define(std::move(munit)));
235 }
236
lookup_in_jd(const char * func_name,LLVMOrcJITDylibRef jd)237 static void *lookup_in_jd(
238 const char *func_name,
239 LLVMOrcJITDylibRef jd) {
240 using llvm::orc::JITDylib;
241 using llvm::JITEvaluatedSymbol;
242 using llvm::orc::ExecutorAddr;
243 JITDylib* JD = ::unwrap(jd);
244 LPJit* jit = get_instance();
245 jit->lookup_mutex.lock();
246 auto func = ExitOnErr(jit->lljit->lookup(*JD, func_name));
247 jit->lookup_mutex.unlock();
248 #if LLVM_VERSION_MAJOR >= 15
249 return func.toPtr<void *>();
250 #else
251 return (void *)(func.getAddress());
252 #endif
253 }
254
remove_jd(LLVMOrcJITDylibRef jd)255 static void remove_jd(LLVMOrcJITDylibRef jd) {
256 using llvm::orc::ExecutionSession;
257 using llvm::orc::JITDylib;
258 auto& es = LPJit::get_instance()->lljit->getExecutionSession();
259 ExitOnErr(es.removeJITDylib(* ::unwrap(jd)));
260 }
261
set_object_cache(llvm::ObjectCache * objcache)262 static void set_object_cache(llvm::ObjectCache *objcache) {
263 auto &ircl = LPJit::get_instance()->lljit->getIRCompileLayer();
264 auto &irc = ircl.getCompiler();
265 auto &sc = dynamic_cast<llvm::orc::SimpleCompiler &>(irc);
266 sc.setObjectCache(objcache);
267 }
268 LLVMTargetMachineRef tm;
269
270 private:
271 LPJit();
272 ~LPJit() = default;
273 LPJit(const LPJit&) = delete;
274 LPJit& operator=(const LPJit&) = delete;
275
276 friend void lpjit_exit();
277
278 static void init_native_targets();
279 llvm::orc::JITTargetMachineBuilder create_jtdb();
280
init_lpjit()281 static void init_lpjit() {
282 jit = new LPJit;
283 std::atexit(lpjit_exit);
284 }
285 static LPJit* jit;
286
287 std::unique_ptr<llvm::orc::LLJIT> lljit;
288 std::unique_ptr<llvm::TargetMachine> tm_unique;
289 /* avoid name conflict */
290 unsigned jit_dylib_count;
291
292 std::mutex lookup_mutex;
293
294 #if DEBUG
295 /* map from module name to gallivm_state */
296 llvm::StringMap<gallivm_state *> gallivm_modules;
297 #endif
298 };
299
300 LPJit* LPJit::jit = NULL;
301
lpjit_exit()302 void lpjit_exit()
303 {
304 delete LPJit::jit;
305 }
306
module_transform(void * Ctx,LLVMModuleRef mod)307 LLVMErrorRef module_transform(void *Ctx, LLVMModuleRef mod) {
308 struct lp_passmgr *mgr;
309
310 lp_passmgr_create(mod, &mgr);
311
312 lp_passmgr_run(mgr, mod,
313 LPJit::get_instance()->tm,
314 get_module_name(mod));
315
316 lp_passmgr_dispose(mgr);
317 return LLVMErrorSuccess;
318 }
319
module_transform_wrapper(void * Ctx,LLVMOrcThreadSafeModuleRef * ModInOut,LLVMOrcMaterializationResponsibilityRef MR)320 LLVMErrorRef module_transform_wrapper(
321 void *Ctx, LLVMOrcThreadSafeModuleRef *ModInOut,
322 LLVMOrcMaterializationResponsibilityRef MR) {
323 return LLVMOrcThreadSafeModuleWithModuleDo(*ModInOut, *module_transform, Ctx);
324 }
325
LPJit()326 LPJit::LPJit() :jit_dylib_count(0) {
327 using namespace llvm::orc;
328
329 lp_init_env_options();
330
331 init_native_targets();
332 JITTargetMachineBuilder JTMB = create_jtdb();
333 tm_unique = ExitOnErr(JTMB.createTargetMachine());
334 tm = wrap(tm_unique.get());
335
336 /* Create an LLJIT instance with an ObjectLinkingLayer (JITLINK)
337 * or RuntimeDyld as the base layer.
338 * intel & perf listeners are not supported by ObjectLinkingLayer yet
339 */
340 lljit = ExitOnErr(
341 LLJITBuilder()
342 .setJITTargetMachineBuilder(std::move(JTMB))
343 #ifdef USE_JITLINK
344 .setObjectLinkingLayerCreator(
345 [&](ExecutionSession &ES, const llvm::Triple &TT) {
346 return std::make_unique<ObjectLinkingLayer>(
347 ES, ExitOnErr(llvm::jitlink::InProcessMemoryManager::Create()));
348 })
349 #else
350 #if LLVM_USE_INTEL_JITEVENTS
351 .RegisterJITEventListener(
352 llvm::JITEventListener::createIntelJITEventListener())
353 #endif
354 #endif
355 .create());
356
357 LLVMOrcIRTransformLayerRef TL = wrap(&lljit->getIRTransformLayer());
358 LLVMOrcIRTransformLayerSetTransform(TL, *module_transform_wrapper, NULL);
359 }
360
init_native_targets()361 void LPJit::init_native_targets() {
362
363 lp_bld_init_native_targets();
364
365 lp_build_init_native_width();
366
367 lp_bld_ppc_disable_denorms();
368 }
369
create_jtdb()370 llvm::orc::JITTargetMachineBuilder LPJit::create_jtdb() {
371 using namespace llvm;
372 using orc::JITTargetMachineBuilder;
373
374 #if defined(_WIN32) && LLVM_VERSION_MAJOR < 15
375 /*
376 * JITLink works on Windows, but only through ELF object format.
377 *
378 * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has
379 * different strings for MinGW/MSVC, so better play it safe and be
380 * explicit.
381 */
382 # ifdef _WIN64
383 JITTargetMachineBuilder JTMB((Triple("x86_64-pc-win32-elf")));
384 # else
385 JITTargetMachineBuilder JTMB((Triple("i686-pc-win32-elf")));
386 # endif
387 #else
388 /*
389 * llvm::sys::getProcessTriple() is bogus. It returns the host LLVM was
390 * compiled on. Be careful when doing cross compilation
391 */
392 JITTargetMachineBuilder JTMB((Triple(sys::getProcessTriple())));
393 #endif
394
395 TargetOptions options;
396 /**
397 * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and
398 * friends for configuring code generation options, like stack alignment.
399 */
400 #if DETECT_ARCH_X86 == 1 && LLVM_VERSION_MAJOR < 13
401 options.StackAlignmentOverride = 4;
402 #endif
403
404 #if DETECT_ARCH_RISCV64 == 1
405 #if defined(__riscv_float_abi_soft)
406 options.MCOptions.ABIName = "lp64";
407 #elif defined(__riscv_float_abi_single)
408 options.MCOptions.ABIName = "lp64f";
409 #elif defined(__riscv_float_abi_double)
410 options.MCOptions.ABIName = "lp64d";
411 #else
412 #error "GALLIVM: unknown target riscv float abi"
413 #endif
414 #endif
415
416 #if DETECT_ARCH_RISCV32 == 1
417 #if defined(__riscv_float_abi_soft)
418 options.MCOptions.ABIName = "ilp32";
419 #elif defined(__riscv_float_abi_single)
420 options.MCOptions.ABIName = "ilp32f";
421 #elif defined(__riscv_float_abi_double)
422 options.MCOptions.ABIName = "ilp32d";
423 #else
424 #error "GALLIVM: unknown target riscv float abi"
425 #endif
426 #endif
427
428 #if DETECT_ARCH_LOONGARCH64 == 1
429 #if defined(__loongarch_lp64) && defined(__loongarch_double_float)
430 options.MCOptions.ABIName = "lp64d";
431 #else
432 #error "GALLIVM: unknown target loongarch float abi"
433 #endif
434 #endif
435
436 JTMB.setOptions(options);
437
438 std::vector<std::string> MAttrs;
439
440 lp_build_fill_mattrs(MAttrs);
441
442 JTMB.addFeatures(MAttrs);
443
444 lp_build_dump_mattrs(MAttrs);
445
446 std::string MCPU = llvm::sys::getHostCPUName().str();
447 /*
448 * Note that the MAttrs set above will be sort of ignored (since we should
449 * not set any which would not be set by specifying the cpu anyway).
450 * It ought to be safe though since getHostCPUName() should include bits
451 * not only from the cpu but environment as well (for instance if it's safe
452 * to use avx instructions which need OS support). According to
453 * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this
454 * right it may be necessary to specify older cpu (or disable mattrs) though
455 * when not using MCJIT so no instructions are generated which the old JIT
456 * can't handle. Not entirely sure if we really need to do anything yet.
457 *
458 * Not sure if the above is also the case for ORCJIT, but we need set CPU
459 * manually since we don't use JITTargetMachineBuilder::detectHost()
460 */
461
462 #if DETECT_ARCH_PPC_64 == 1
463 /*
464 * Large programs, e.g. gnome-shell and firefox, may tax the addressability
465 * of the Medium code model once dynamically generated JIT-compiled shader
466 * programs are linked in and relocated. Yet the default code model as of
467 * LLVM 8 is Medium or even Small.
468 * The cost of changing from Medium to Large is negligible:
469 * - an additional 8-byte pointer stored immediately before the shader entrypoint;
470 * - change an add-immediate (addis) instruction to a load (ld).
471 */
472 JTMB.setCodeModel(CodeModel::Large);
473
474 #if UTIL_ARCH_LITTLE_ENDIAN
475 /*
476 * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL",
477 * resulting in (big-endian) "generic" being returned on
478 * little-endian Power8NVL systems. The result was that code that
479 * attempted to load the least significant 32 bits of a 64-bit quantity
480 * from memory loaded the wrong half. This resulted in failures in some
481 * Piglit tests, e.g.
482 * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint
483 */
484 if (MCPU == "generic")
485 MCPU = "pwr8";
486 #endif
487 #endif
488
489 #if DETECT_ARCH_MIPS64 == 1
490 /*
491 * ls3a4000 CPU and ls2k1000 SoC is a mips64r5 compatible with MSA SIMD
492 * instruction set implemented, while ls3a3000 is mips64r2 compatible
493 * only. getHostCPUName() return "generic" on all loongson
494 * mips CPU currently. So we override the MCPU to mips64r5 if MSA is
495 * implemented, feedback to mips64r2 for all other ordinary mips64 cpu.
496 */
497 if (MCPU == "generic")
498 MCPU = util_get_cpu_caps()->has_msa ? "mips64r5" : "mips64r2";
499 #endif
500
501 #if DETECT_ARCH_RISCV64 == 1
502 /**
503 * should be fixed with https://reviews.llvm.org/D121149 in llvm 15,
504 * set it anyway for llvm 14
505 */
506 if (MCPU == "generic")
507 MCPU = "generic-rv64";
508
509 JTMB.setCodeModel(CodeModel::Medium);
510 JTMB.setRelocationModel(Reloc::PIC_);
511 #endif
512
513 #if DETECT_ARCH_RISCV32 == 1
514 /**
515 * should be fixed with https://reviews.llvm.org/D121149 in llvm 15,
516 * set it anyway for llvm 14
517 */
518 if (MCPU == "generic")
519 MCPU = "generic-rv32";
520
521 JTMB.setCodeModel(CodeModel::Medium);
522 JTMB.setRelocationModel(Reloc::PIC_);
523 #endif
524
525 JTMB.setCPU(MCPU);
526 if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
527 debug_printf("llc -mcpu option: %s\n", MCPU.c_str());
528 }
529
530 return JTMB;
531 }
532
533
534 } /* Anonymous namespace */
535
536 bool
lp_build_init(void)537 lp_build_init(void)
538 {
539 (void)LPJit::get_instance();
540 return true;
541 }
542
543 bool
init_gallivm_state(struct gallivm_state * gallivm,const char * name,lp_context_ref * context,struct lp_cached_code * cache)544 init_gallivm_state(struct gallivm_state *gallivm, const char *name,
545 lp_context_ref *context, struct lp_cached_code *cache)
546 {
547 assert(!gallivm->context);
548 assert(!gallivm->_ts_context);
549 assert(!gallivm->module);
550
551 if (!lp_build_init())
552 return false;
553
554 gallivm->cache = cache;
555
556 gallivm->_ts_context = context->ref;
557 gallivm->context = LLVMOrcThreadSafeContextGetContext(context->ref);
558
559 gallivm->module_name = LPJit::get_unique_name(name);
560 gallivm->module = LLVMModuleCreateWithNameInContext(gallivm->module_name,
561 gallivm->context);
562 #if DETECT_ARCH_X86 == 1
563 lp_set_module_stack_alignment_override(gallivm->module, 4);
564 #endif
565 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
566 gallivm->_per_module_jd = LPJit::create_jit_dylib(gallivm->module_name);
567
568 gallivm->target = LLVMCreateTargetDataLayout(LPJit::get_instance()->tm);
569
570 lp_build_coro_declare_malloc_hooks(gallivm);
571 return true;
572 }
573
574 struct gallivm_state *
gallivm_create(const char * name,lp_context_ref * context,struct lp_cached_code * cache)575 gallivm_create(const char *name, lp_context_ref *context,
576 struct lp_cached_code *cache){
577 struct gallivm_state *gallivm;
578
579 gallivm = CALLOC_STRUCT(gallivm_state);
580 if (gallivm) {
581 if (!init_gallivm_state(gallivm, name, context, cache)) {
582 FREE(gallivm);
583 gallivm = NULL;
584 }
585 }
586
587 assert(gallivm != NULL);
588 return gallivm;
589 }
590
591 void
gallivm_destroy(struct gallivm_state * gallivm)592 gallivm_destroy(struct gallivm_state *gallivm)
593 {
594 LPJit::remove_jd(gallivm->_per_module_jd);
595 gallivm->_per_module_jd = nullptr;
596 FREE(gallivm);
597 }
598
599 void
gallivm_free_ir(struct gallivm_state * gallivm)600 gallivm_free_ir(struct gallivm_state *gallivm)
601 {
602 if (gallivm->module)
603 LLVMDisposeModule(gallivm->module);
604 FREE(gallivm->module_name);
605
606 if (gallivm->target) {
607 LLVMDisposeTargetData(gallivm->target);
608 }
609
610 if (gallivm->builder)
611 LLVMDisposeBuilder(gallivm->builder);
612
613 if (gallivm->cache) {
614 if (gallivm->cache->jit_obj_cache)
615 lp_free_objcache(gallivm->cache->jit_obj_cache);
616 free(gallivm->cache->data);
617 }
618
619 gallivm->target = NULL;
620 gallivm->module=NULL;
621 gallivm->module_name=NULL;
622 gallivm->builder=NULL;
623 gallivm->context=NULL;
624 gallivm->_ts_context=NULL;
625 gallivm->cache=NULL;
626 LPJit::deregister_gallivm_state(gallivm);
627 LPJit::set_object_cache(NULL);
628 }
629
630 void
gallivm_add_global_mapping(struct gallivm_state * gallivm,LLVMValueRef sym,void * addr)631 gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr)
632 {
633 LPJit::add_mapping_to_jd(sym, addr, gallivm->_per_module_jd);
634 }
635
636 void
gallivm_compile_module(struct gallivm_state * gallivm)637 gallivm_compile_module(struct gallivm_state *gallivm)
638 {
639 lp_init_printf_hook(gallivm);
640 gallivm_add_global_mapping(gallivm, gallivm->debug_printf_hook,
641 (void *)debug_printf);
642
643 lp_init_clock_hook(gallivm);
644 gallivm_add_global_mapping(gallivm, gallivm->get_time_hook,
645 (void *)os_time_get_nano);
646
647 lp_build_coro_add_malloc_hooks(gallivm);
648
649 LPJit::add_ir_module_to_jd(gallivm->_ts_context, gallivm->module,
650 gallivm->_per_module_jd);
651 /* ownership of module is now transferred into orc jit,
652 * disallow modifying it
653 */
654 LPJit::register_gallivm_state(gallivm);
655 gallivm->module = nullptr;
656
657 if (gallivm->cache) {
658 if (!gallivm->cache->jit_obj_cache) {
659 LPObjectCacheORC *objcache = new LPObjectCacheORC(gallivm->cache);
660 gallivm->cache->jit_obj_cache = (void *)objcache;
661 }
662 auto *objcache = (LPObjectCacheORC *)gallivm->cache->jit_obj_cache;
663 LPJit::set_object_cache(objcache);
664 }
665 /* defer compilation till first lookup by gallivm_jit_function */
666 }
667
668 func_pointer
gallivm_jit_function(struct gallivm_state * gallivm,LLVMValueRef func,const char * func_name)669 gallivm_jit_function(struct gallivm_state *gallivm,
670 LLVMValueRef func, const char *func_name)
671 {
672 return pointer_to_func(
673 LPJit::lookup_in_jd(func_name, gallivm->_per_module_jd));
674 }
675
676 void
gallivm_stub_func(struct gallivm_state * gallivm,LLVMValueRef func)677 gallivm_stub_func(struct gallivm_state *gallivm, LLVMValueRef func)
678 {
679 /*
680 * ORCJIT cannot accept a function with absolutely no content at all.
681 * Generate a "void func() {}" stub here.
682 */
683 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context,
684 func, "entry");
685 LLVMBuilderRef builder = gallivm->builder;
686 assert(builder);
687 LLVMPositionBuilderAtEnd(builder, block);
688 LLVMBuildRetVoid(builder);
689 }
690