1 //===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
11 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
12 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
13
14 #include "llvm/Object/COFF.h"
15
16 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
17
18 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
19
20 #define DEBUG_TYPE "orc"
21
22 using namespace llvm;
23 using namespace llvm::orc;
24 using namespace llvm::orc::shared;
25
26 namespace llvm {
27 namespace orc {
28 namespace shared {
29
30 using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
31 using SPSCOFFJITDylibDepInfoMap =
32 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
33 using SPSCOFFObjectSectionsMap =
34 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
35 using SPSCOFFRegisterObjectSectionsArgs =
36 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
37 using SPSCOFFDeregisterObjectSectionsArgs =
38 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
39
40 } // namespace shared
41 } // namespace orc
42 } // namespace llvm
43 namespace {
44
45 class COFFHeaderMaterializationUnit : public MaterializationUnit {
46 public:
COFFHeaderMaterializationUnit(COFFPlatform & CP,const SymbolStringPtr & HeaderStartSymbol)47 COFFHeaderMaterializationUnit(COFFPlatform &CP,
48 const SymbolStringPtr &HeaderStartSymbol)
49 : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)),
50 CP(CP) {}
51
getName() const52 StringRef getName() const override { return "COFFHeaderMU"; }
53
materialize(std::unique_ptr<MaterializationResponsibility> R)54 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
55 unsigned PointerSize;
56 support::endianness Endianness;
57 const auto &TT =
58 CP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
59
60 switch (TT.getArch()) {
61 case Triple::x86_64:
62 PointerSize = 8;
63 Endianness = support::endianness::little;
64 break;
65 default:
66 llvm_unreachable("Unrecognized architecture");
67 }
68
69 auto G = std::make_unique<jitlink::LinkGraph>(
70 "<COFFHeaderMU>", TT, PointerSize, Endianness,
71 jitlink::getGenericEdgeKindName);
72 auto &HeaderSection = G->createSection("__header", MemProt::Read);
73 auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
74
75 // Init symbol is __ImageBase symbol.
76 auto &ImageBaseSymbol = G->addDefinedSymbol(
77 HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(),
78 jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
79
80 addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol);
81
82 CP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
83 }
84
discard(const JITDylib & JD,const SymbolStringPtr & Sym)85 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
86
87 private:
88 struct HeaderSymbol {
89 const char *Name;
90 uint64_t Offset;
91 };
92
93 struct NTHeader {
94 support::ulittle32_t PEMagic;
95 object::coff_file_header FileHeader;
96 struct PEHeader {
97 object::pe32plus_header Header;
98 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
99 } OptionalHeader;
100 };
101
102 struct HeaderBlockContent {
103 object::dos_header DOSHeader;
104 COFFHeaderMaterializationUnit::NTHeader NTHeader;
105 };
106
createHeaderBlock(jitlink::LinkGraph & G,jitlink::Section & HeaderSection)107 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
108 jitlink::Section &HeaderSection) {
109 HeaderBlockContent Hdr = {};
110
111 // Set up magic
112 Hdr.DOSHeader.Magic[0] = 'M';
113 Hdr.DOSHeader.Magic[1] = 'Z';
114 Hdr.DOSHeader.AddressOfNewExeHeader =
115 offsetof(HeaderBlockContent, NTHeader);
116 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
117 Hdr.NTHeader.PEMagic = PEMagic;
118 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
119
120 switch (G.getTargetTriple().getArch()) {
121 case Triple::x86_64:
122 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
123 break;
124 default:
125 llvm_unreachable("Unrecognized architecture");
126 }
127
128 auto HeaderContent = G.allocateString(
129 StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
130
131 return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
132 0);
133 }
134
addImageBaseRelocationEdge(jitlink::Block & B,jitlink::Symbol & ImageBase)135 static void addImageBaseRelocationEdge(jitlink::Block &B,
136 jitlink::Symbol &ImageBase) {
137 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
138 offsetof(NTHeader, OptionalHeader) +
139 offsetof(object::pe32plus_header, ImageBase);
140 B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0);
141 }
142
143 static MaterializationUnit::Interface
createHeaderInterface(COFFPlatform & MOP,const SymbolStringPtr & HeaderStartSymbol)144 createHeaderInterface(COFFPlatform &MOP,
145 const SymbolStringPtr &HeaderStartSymbol) {
146 SymbolFlagsMap HeaderSymbolFlags;
147
148 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
149
150 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
151 HeaderStartSymbol);
152 }
153
154 COFFPlatform &CP;
155 };
156
157 } // end anonymous namespace
158
159 namespace llvm {
160 namespace orc {
161
162 Expected<std::unique_ptr<COFFPlatform>>
Create(ExecutionSession & ES,ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,const char * OrcRuntimePath,LoadDynamicLibrary LoadDynLibrary,bool StaticVCRuntime,const char * VCRuntimePath,std::optional<SymbolAliasMap> RuntimeAliases)163 COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
164 JITDylib &PlatformJD, const char *OrcRuntimePath,
165 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
166 const char *VCRuntimePath,
167 std::optional<SymbolAliasMap> RuntimeAliases) {
168 auto &EPC = ES.getExecutorProcessControl();
169
170 // If the target is not supported then bail out immediately.
171 if (!supportedTarget(EPC.getTargetTriple()))
172 return make_error<StringError>("Unsupported COFFPlatform triple: " +
173 EPC.getTargetTriple().str(),
174 inconvertibleErrorCode());
175
176 // Create default aliases if the caller didn't supply any.
177 if (!RuntimeAliases)
178 RuntimeAliases = standardPlatformAliases(ES);
179
180 // Define the aliases.
181 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
182 return std::move(Err);
183
184 auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>");
185
186 // Add JIT-dispatch function support symbols.
187 if (auto Err = HostFuncJD.define(absoluteSymbols(
188 {{ES.intern("__orc_rt_jit_dispatch"),
189 {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
190 JITSymbolFlags::Exported}},
191 {ES.intern("__orc_rt_jit_dispatch_ctx"),
192 {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
193 JITSymbolFlags::Exported}}})))
194 return std::move(Err);
195
196 PlatformJD.addToLinkOrder(HostFuncJD);
197
198 // Create the instance.
199 Error Err = Error::success();
200 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
201 ES, ObjLinkingLayer, PlatformJD, OrcRuntimePath,
202 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err));
203 if (Err)
204 return std::move(Err);
205 return std::move(P);
206 }
207
getPerJDObjectFile()208 Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
209 auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");
210 if (!PerJDObj)
211 return PerJDObj.takeError();
212
213 if (!*PerJDObj)
214 return make_error<StringError>("Could not find per jd object file",
215 inconvertibleErrorCode());
216
217 auto Buffer = (*PerJDObj)->getAsBinary();
218 if (!Buffer)
219 return Buffer.takeError();
220
221 return (*Buffer)->getMemoryBufferRef();
222 }
223
addAliases(ExecutionSession & ES,SymbolAliasMap & Aliases,ArrayRef<std::pair<const char *,const char * >> AL)224 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
225 ArrayRef<std::pair<const char *, const char *>> AL) {
226 for (auto &KV : AL) {
227 auto AliasName = ES.intern(KV.first);
228 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
229 Aliases[std::move(AliasName)] = {ES.intern(KV.second),
230 JITSymbolFlags::Exported};
231 }
232 }
233
setupJITDylib(JITDylib & JD)234 Error COFFPlatform::setupJITDylib(JITDylib &JD) {
235 if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(
236 *this, COFFHeaderStartSymbol)))
237 return Err;
238
239 if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())
240 return Err;
241
242 // Define the CXX aliases.
243 SymbolAliasMap CXXAliases;
244 addAliases(ES, CXXAliases, requiredCXXAliases());
245 if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))
246 return Err;
247
248 auto PerJDObj = getPerJDObjectFile();
249 if (!PerJDObj)
250 return PerJDObj.takeError();
251
252 auto I = getObjectFileInterface(ES, *PerJDObj);
253 if (!I)
254 return I.takeError();
255
256 if (auto Err = ObjLinkingLayer.add(
257 JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))
258 return Err;
259
260 if (!Bootstrapping) {
261 auto ImportedLibs = StaticVCRuntime
262 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
263 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
264 if (!ImportedLibs)
265 return ImportedLibs.takeError();
266 for (auto &Lib : *ImportedLibs)
267 if (auto Err = LoadDynLibrary(JD, Lib))
268 return Err;
269 if (StaticVCRuntime)
270 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
271 return Err;
272 }
273
274 JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));
275 return Error::success();
276 }
277
teardownJITDylib(JITDylib & JD)278 Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
279 std::lock_guard<std::mutex> Lock(PlatformMutex);
280 auto I = JITDylibToHeaderAddr.find(&JD);
281 if (I != JITDylibToHeaderAddr.end()) {
282 assert(HeaderAddrToJITDylib.count(I->second) &&
283 "HeaderAddrToJITDylib missing entry");
284 HeaderAddrToJITDylib.erase(I->second);
285 JITDylibToHeaderAddr.erase(I);
286 }
287 return Error::success();
288 }
289
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)290 Error COFFPlatform::notifyAdding(ResourceTracker &RT,
291 const MaterializationUnit &MU) {
292 auto &JD = RT.getJITDylib();
293 const auto &InitSym = MU.getInitializerSymbol();
294 if (!InitSym)
295 return Error::success();
296
297 RegisteredInitSymbols[&JD].add(InitSym,
298 SymbolLookupFlags::WeaklyReferencedSymbol);
299
300 LLVM_DEBUG({
301 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
302 << MU.getName() << "\n";
303 });
304 return Error::success();
305 }
306
notifyRemoving(ResourceTracker & RT)307 Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
308 llvm_unreachable("Not supported yet");
309 }
310
standardPlatformAliases(ExecutionSession & ES)311 SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
312 SymbolAliasMap Aliases;
313 addAliases(ES, Aliases, standardRuntimeUtilityAliases());
314 return Aliases;
315 }
316
317 ArrayRef<std::pair<const char *, const char *>>
requiredCXXAliases()318 COFFPlatform::requiredCXXAliases() {
319 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
320 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
321 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
322 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
323
324 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
325 }
326
327 ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases()328 COFFPlatform::standardRuntimeUtilityAliases() {
329 static const std::pair<const char *, const char *>
330 StandardRuntimeUtilityAliases[] = {
331 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
332 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
333 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
334 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
335 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
336 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
337
338 return ArrayRef<std::pair<const char *, const char *>>(
339 StandardRuntimeUtilityAliases);
340 }
341
supportedTarget(const Triple & TT)342 bool COFFPlatform::supportedTarget(const Triple &TT) {
343 switch (TT.getArch()) {
344 case Triple::x86_64:
345 return true;
346 default:
347 return false;
348 }
349 }
350
COFFPlatform(ExecutionSession & ES,ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,const char * OrcRuntimePath,LoadDynamicLibrary LoadDynLibrary,bool StaticVCRuntime,const char * VCRuntimePath,Error & Err)351 COFFPlatform::COFFPlatform(ExecutionSession &ES,
352 ObjectLinkingLayer &ObjLinkingLayer,
353 JITDylib &PlatformJD, const char *OrcRuntimePath,
354 LoadDynamicLibrary LoadDynLibrary,
355 bool StaticVCRuntime, const char *VCRuntimePath,
356 Error &Err)
357 : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
358 LoadDynLibrary(std::move(LoadDynLibrary)),
359 StaticVCRuntime(StaticVCRuntime),
360 COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
361 ErrorAsOutParameter _(&Err);
362
363 // Create a generator for the ORC runtime archive.
364 auto OrcRuntimeArchiveGenerator =
365 StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
366 if (!OrcRuntimeArchiveGenerator) {
367 Err = OrcRuntimeArchiveGenerator.takeError();
368 return;
369 }
370
371 auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);
372 if (!ArchiveBuffer) {
373 Err = createFileError(OrcRuntimePath, ArchiveBuffer.getError());
374 return;
375 }
376 OrcRuntimeArchiveBuffer = std::move(*ArchiveBuffer);
377 OrcRuntimeArchive =
378 std::make_unique<object::Archive>(*OrcRuntimeArchiveBuffer, Err);
379 if (Err)
380 return;
381
382 Bootstrapping.store(true);
383 ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
384
385 // Load vc runtime
386 auto VCRT =
387 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);
388 if (!VCRT) {
389 Err = VCRT.takeError();
390 return;
391 }
392 VCRuntimeBootstrap = std::move(*VCRT);
393
394 for (auto &Lib : (*OrcRuntimeArchiveGenerator)->getImportedDynamicLibraries())
395 DylibsToPreload.insert(Lib);
396
397 auto ImportedLibs =
398 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)
399 : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);
400 if (!ImportedLibs) {
401 Err = ImportedLibs.takeError();
402 return;
403 }
404
405 for (auto &Lib : *ImportedLibs)
406 DylibsToPreload.insert(Lib);
407
408 PlatformJD.addGenerator(std::move(*OrcRuntimeArchiveGenerator));
409
410 // PlatformJD hasn't been set up by the platform yet (since we're creating
411 // the platform now), so set it up.
412 if (auto E2 = setupJITDylib(PlatformJD)) {
413 Err = std::move(E2);
414 return;
415 }
416
417 for (auto& Lib : DylibsToPreload)
418 if (auto E2 = LoadDynLibrary(PlatformJD, Lib)) {
419 Err = std::move(E2);
420 return;
421 }
422
423 if (StaticVCRuntime)
424 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) {
425 Err = std::move(E2);
426 return;
427 }
428
429 // Associate wrapper function tags with JIT-side function implementations.
430 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
431 Err = std::move(E2);
432 return;
433 }
434
435 // Lookup addresses of runtime functions callable by the platform,
436 // call the platform bootstrap function to initialize the platform-state
437 // object in the executor.
438 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
439 Err = std::move(E2);
440 return;
441 }
442
443 Bootstrapping.store(false);
444 JDBootstrapStates.clear();
445 }
446
447 Expected<COFFPlatform::JITDylibDepMap>
buildJDDepMap(JITDylib & JD)448 COFFPlatform::buildJDDepMap(JITDylib &JD) {
449 return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> {
450 JITDylibDepMap JDDepMap;
451
452 SmallVector<JITDylib *, 16> Worklist({&JD});
453 while (!Worklist.empty()) {
454 auto CurJD = Worklist.back();
455 Worklist.pop_back();
456
457 auto &DM = JDDepMap[CurJD];
458 CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
459 DM.reserve(O.size());
460 for (auto &KV : O) {
461 if (KV.first == CurJD)
462 continue;
463 {
464 // Bare jitdylibs not known to the platform
465 std::lock_guard<std::mutex> Lock(PlatformMutex);
466 if (!JITDylibToHeaderAddr.count(KV.first)) {
467 LLVM_DEBUG({
468 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
469 "LinkOrder: "
470 << CurJD->getName() << "\n";
471 });
472 continue;
473 }
474 }
475 DM.push_back(KV.first);
476 // Push unvisited entry.
477 if (!JDDepMap.count(KV.first)) {
478 Worklist.push_back(KV.first);
479 JDDepMap[KV.first] = {};
480 }
481 }
482 });
483 }
484 return std::move(JDDepMap);
485 });
486 }
487
pushInitializersLoop(PushInitializersSendResultFn SendResult,JITDylibSP JD,JITDylibDepMap & JDDepMap)488 void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
489 JITDylibSP JD,
490 JITDylibDepMap &JDDepMap) {
491 SmallVector<JITDylib *, 16> Worklist({JD.get()});
492 DenseSet<JITDylib *> Visited({JD.get()});
493 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
494 ES.runSessionLocked([&]() {
495 while (!Worklist.empty()) {
496 auto CurJD = Worklist.back();
497 Worklist.pop_back();
498
499 auto RISItr = RegisteredInitSymbols.find(CurJD);
500 if (RISItr != RegisteredInitSymbols.end()) {
501 NewInitSymbols[CurJD] = std::move(RISItr->second);
502 RegisteredInitSymbols.erase(RISItr);
503 }
504
505 for (auto *DepJD : JDDepMap[CurJD])
506 if (!Visited.count(DepJD)) {
507 Worklist.push_back(DepJD);
508 Visited.insert(DepJD);
509 }
510 }
511 });
512
513 // If there are no further init symbols to look up then send the link order
514 // (as a list of header addresses) to the caller.
515 if (NewInitSymbols.empty()) {
516 // Build the dep info map to return.
517 COFFJITDylibDepInfoMap DIM;
518 DIM.reserve(JDDepMap.size());
519 for (auto &KV : JDDepMap) {
520 std::lock_guard<std::mutex> Lock(PlatformMutex);
521 COFFJITDylibDepInfo DepInfo;
522 DepInfo.reserve(KV.second.size());
523 for (auto &Dep : KV.second) {
524 DepInfo.push_back(JITDylibToHeaderAddr[Dep]);
525 }
526 auto H = JITDylibToHeaderAddr[KV.first];
527 DIM.push_back(std::make_pair(H, std::move(DepInfo)));
528 }
529 SendResult(DIM);
530 return;
531 }
532
533 // Otherwise issue a lookup and re-run this phase when it completes.
534 lookupInitSymbolsAsync(
535 [this, SendResult = std::move(SendResult), &JD,
536 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
537 if (Err)
538 SendResult(std::move(Err));
539 else
540 pushInitializersLoop(std::move(SendResult), JD, JDDepMap);
541 },
542 ES, std::move(NewInitSymbols));
543 }
544
rt_pushInitializers(PushInitializersSendResultFn SendResult,ExecutorAddr JDHeaderAddr)545 void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
546 ExecutorAddr JDHeaderAddr) {
547 JITDylibSP JD;
548 {
549 std::lock_guard<std::mutex> Lock(PlatformMutex);
550 auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
551 if (I != HeaderAddrToJITDylib.end())
552 JD = I->second;
553 }
554
555 LLVM_DEBUG({
556 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
557 if (JD)
558 dbgs() << "pushing initializers for " << JD->getName() << "\n";
559 else
560 dbgs() << "No JITDylib for header address.\n";
561 });
562
563 if (!JD) {
564 SendResult(
565 make_error<StringError>("No JITDylib with header addr " +
566 formatv("{0:x}", JDHeaderAddr.getValue()),
567 inconvertibleErrorCode()));
568 return;
569 }
570
571 auto JDDepMap = buildJDDepMap(*JD);
572 if (!JDDepMap) {
573 SendResult(JDDepMap.takeError());
574 return;
575 }
576
577 pushInitializersLoop(std::move(SendResult), JD, *JDDepMap);
578 }
579
rt_lookupSymbol(SendSymbolAddressFn SendResult,ExecutorAddr Handle,StringRef SymbolName)580 void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
581 ExecutorAddr Handle, StringRef SymbolName) {
582 LLVM_DEBUG({
583 dbgs() << "COFFPlatform::rt_lookupSymbol(\""
584 << formatv("{0:x}", Handle.getValue()) << "\")\n";
585 });
586
587 JITDylib *JD = nullptr;
588
589 {
590 std::lock_guard<std::mutex> Lock(PlatformMutex);
591 auto I = HeaderAddrToJITDylib.find(Handle);
592 if (I != HeaderAddrToJITDylib.end())
593 JD = I->second;
594 }
595
596 if (!JD) {
597 LLVM_DEBUG({
598 dbgs() << " No JITDylib for handle "
599 << formatv("{0:x}", Handle.getValue()) << "\n";
600 });
601 SendResult(make_error<StringError>("No JITDylib associated with handle " +
602 formatv("{0:x}", Handle.getValue()),
603 inconvertibleErrorCode()));
604 return;
605 }
606
607 // Use functor class to work around XL build compiler issue on AIX.
608 class RtLookupNotifyComplete {
609 public:
610 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
611 : SendResult(std::move(SendResult)) {}
612 void operator()(Expected<SymbolMap> Result) {
613 if (Result) {
614 assert(Result->size() == 1 && "Unexpected result map count");
615 SendResult(ExecutorAddr(Result->begin()->second.getAddress()));
616 } else {
617 SendResult(Result.takeError());
618 }
619 }
620
621 private:
622 SendSymbolAddressFn SendResult;
623 };
624
625 ES.lookup(
626 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
627 SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
628 RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
629 }
630
associateRuntimeSupportFunctions(JITDylib & PlatformJD)631 Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
632 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
633
634 using LookupSymbolSPSSig =
635 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
636 WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] =
637 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
638 &COFFPlatform::rt_lookupSymbol);
639 using PushInitializersSPSSig =
640 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
641 WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] =
642 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
643 this, &COFFPlatform::rt_pushInitializers);
644
645 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
646 }
647
runBootstrapInitializers(JDBootstrapState & BState)648 Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
649 llvm::sort(BState.Initializers);
650 if (auto Err =
651 runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ"))
652 return Err;
653
654 if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init"))
655 return Err;
656
657 if (auto Err =
658 runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ"))
659 return Err;
660 return Error::success();
661 }
662
runBootstrapSubsectionInitializers(JDBootstrapState & BState,StringRef Start,StringRef End)663 Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
664 StringRef Start,
665 StringRef End) {
666 for (auto &Initializer : BState.Initializers)
667 if (Initializer.first >= Start && Initializer.first <= End &&
668 Initializer.second) {
669 auto Res =
670 ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second);
671 if (!Res)
672 return Res.takeError();
673 }
674 return Error::success();
675 }
676
bootstrapCOFFRuntime(JITDylib & PlatformJD)677 Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
678 // Lookup of runtime symbols causes the collection of initializers if
679 // it's static linking setting.
680 if (auto Err = lookupAndRecordAddrs(
681 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
682 {
683 {ES.intern("__orc_rt_coff_platform_bootstrap"),
684 &orc_rt_coff_platform_bootstrap},
685 {ES.intern("__orc_rt_coff_platform_shutdown"),
686 &orc_rt_coff_platform_shutdown},
687 {ES.intern("__orc_rt_coff_register_jitdylib"),
688 &orc_rt_coff_register_jitdylib},
689 {ES.intern("__orc_rt_coff_deregister_jitdylib"),
690 &orc_rt_coff_deregister_jitdylib},
691 {ES.intern("__orc_rt_coff_register_object_sections"),
692 &orc_rt_coff_register_object_sections},
693 {ES.intern("__orc_rt_coff_deregister_object_sections"),
694 &orc_rt_coff_deregister_object_sections},
695 }))
696 return Err;
697
698 // Call bootstrap functions
699 if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))
700 return Err;
701
702 // Do the pending jitdylib registration actions that we couldn't do
703 // because orc runtime was not linked fully.
704 for (auto KV : JDBootstrapStates) {
705 auto &JDBState = KV.second;
706 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
707 orc_rt_coff_register_jitdylib, JDBState.JDName,
708 JDBState.HeaderAddr))
709 return Err;
710
711 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
712 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
713 SPSCOFFObjectSectionsMap, bool)>(
714 orc_rt_coff_register_object_sections, JDBState.HeaderAddr,
715 ObjSectionMap, false))
716 return Err;
717 }
718
719 // Run static initializers collected in bootstrap stage.
720 for (auto KV : JDBootstrapStates) {
721 auto &JDBState = KV.second;
722 if (auto Err = runBootstrapInitializers(JDBState))
723 return Err;
724 }
725
726 return Error::success();
727 }
728
runSymbolIfExists(JITDylib & PlatformJD,StringRef SymbolName)729 Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
730 StringRef SymbolName) {
731 ExecutorAddr jit_function;
732 auto AfterCLookupErr = lookupAndRecordAddrs(
733 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
734 {{ES.intern(SymbolName), &jit_function}});
735 if (!AfterCLookupErr) {
736 auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function);
737 if (!Res)
738 return Res.takeError();
739 return Error::success();
740 }
741 if (!AfterCLookupErr.isA<SymbolsNotFound>())
742 return AfterCLookupErr;
743 consumeError(std::move(AfterCLookupErr));
744 return Error::success();
745 }
746
modifyPassConfig(MaterializationResponsibility & MR,jitlink::LinkGraph & LG,jitlink::PassConfiguration & Config)747 void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
748 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
749 jitlink::PassConfiguration &Config) {
750
751 bool IsBootstrapping = CP.Bootstrapping.load();
752
753 if (auto InitSymbol = MR.getInitializerSymbol()) {
754 if (InitSymbol == CP.COFFHeaderStartSymbol) {
755 Config.PostAllocationPasses.push_back(
756 [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
757 return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping);
758 });
759 return;
760 }
761 Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
762 return preserveInitializerSections(G, MR);
763 });
764 }
765
766 if (!IsBootstrapping)
767 Config.PostFixupPasses.push_back(
768 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
769 return registerObjectPlatformSections(G, JD);
770 });
771 else
772 Config.PostFixupPasses.push_back(
773 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
774 return registerObjectPlatformSectionsInBootstrap(G, JD);
775 });
776 }
777
778 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
getSyntheticSymbolDependencies(MaterializationResponsibility & MR)779 COFFPlatform::COFFPlatformPlugin::getSyntheticSymbolDependencies(
780 MaterializationResponsibility &MR) {
781 std::lock_guard<std::mutex> Lock(PluginMutex);
782 auto I = InitSymbolDeps.find(&MR);
783 if (I != InitSymbolDeps.end()) {
784 SyntheticSymbolDependenciesMap Result;
785 Result[MR.getInitializerSymbol()] = std::move(I->second);
786 InitSymbolDeps.erase(&MR);
787 return Result;
788 }
789 return SyntheticSymbolDependenciesMap();
790 }
791
associateJITDylibHeaderSymbol(jitlink::LinkGraph & G,MaterializationResponsibility & MR,bool IsBootstraping)792 Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
793 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
794 bool IsBootstraping) {
795 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
796 return Sym->getName() == *CP.COFFHeaderStartSymbol;
797 });
798 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
799
800 auto &JD = MR.getTargetJITDylib();
801 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
802 auto HeaderAddr = (*I)->getAddress();
803 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
804 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
805 if (!IsBootstraping) {
806 G.allocActions().push_back(
807 {cantFail(WrapperFunctionCall::Create<
808 SPSArgList<SPSString, SPSExecutorAddr>>(
809 CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)),
810 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
811 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
812 } else {
813 G.allocActions().push_back(
814 {{},
815 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
816 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
817 JDBootstrapState BState;
818 BState.JD = &JD;
819 BState.JDName = JD.getName();
820 BState.HeaderAddr = HeaderAddr;
821 CP.JDBootstrapStates.emplace(&JD, BState);
822 }
823
824 return Error::success();
825 }
826
registerObjectPlatformSections(jitlink::LinkGraph & G,JITDylib & JD)827 Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
828 jitlink::LinkGraph &G, JITDylib &JD) {
829 COFFObjectSectionsMap ObjSecs;
830 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
831 assert(HeaderAddr && "Must be registered jitdylib");
832 for (auto &S : G.sections()) {
833 jitlink::SectionRange Range(S);
834 if (Range.getSize())
835 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
836 }
837
838 G.allocActions().push_back(
839 {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
840 CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)),
841 cantFail(
842 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
843 CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
844 ObjSecs))});
845
846 return Error::success();
847 }
848
preserveInitializerSections(jitlink::LinkGraph & G,MaterializationResponsibility & MR)849 Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
850 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
851 JITLinkSymbolSet InitSectionSymbols;
852 for (auto &Sec : G.sections())
853 if (COFFPlatform::isInitializerSection(Sec.getName()))
854 for (auto *B : Sec.blocks())
855 if (!B->edges_empty())
856 InitSectionSymbols.insert(
857 &G.addAnonymousSymbol(*B, 0, 0, false, true));
858
859 std::lock_guard<std::mutex> Lock(PluginMutex);
860 InitSymbolDeps[&MR] = InitSectionSymbols;
861 return Error::success();
862 }
863
864 Error COFFPlatform::COFFPlatformPlugin::
registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph & G,JITDylib & JD)865 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
866 JITDylib &JD) {
867 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
868 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
869 COFFObjectSectionsMap ObjSecs;
870 for (auto &S : G.sections()) {
871 jitlink::SectionRange Range(S);
872 if (Range.getSize())
873 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
874 }
875
876 G.allocActions().push_back(
877 {{},
878 cantFail(
879 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
880 CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
881 ObjSecs))});
882
883 auto &BState = CP.JDBootstrapStates[&JD];
884 BState.ObjectSectionsMaps.push_back(std::move(ObjSecs));
885
886 // Collect static initializers
887 for (auto &S : G.sections())
888 if (COFFPlatform::isInitializerSection(S.getName()))
889 for (auto *B : S.blocks()) {
890 if (B->edges_empty())
891 continue;
892 for (auto &E : B->edges())
893 BState.Initializers.push_back(std::make_pair(
894 S.getName().str(),
895 ExecutorAddr(E.getTarget().getAddress() + E.getAddend())));
896 }
897
898 return Error::success();
899 }
900
901 } // End namespace orc.
902 } // End namespace llvm.
903