1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
10
11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12 #include "llvm/Support/WindowsError.h"
13
14 #include <algorithm>
15
16 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
17 #include <fcntl.h>
18 #include <sys/mman.h>
19 #include <unistd.h>
20 #elif defined(_WIN32)
21 #include <windows.h>
22 #endif
23
24 namespace llvm {
25 namespace orc {
26
~MemoryMapper()27 MemoryMapper::~MemoryMapper() {}
28
InProcessMemoryMapper(size_t PageSize)29 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize)
30 : PageSize(PageSize) {}
31
32 Expected<std::unique_ptr<InProcessMemoryMapper>>
Create()33 InProcessMemoryMapper::Create() {
34 auto PageSize = sys::Process::getPageSize();
35 if (!PageSize)
36 return PageSize.takeError();
37 return std::make_unique<InProcessMemoryMapper>(*PageSize);
38 }
39
reserve(size_t NumBytes,OnReservedFunction OnReserved)40 void InProcessMemoryMapper::reserve(size_t NumBytes,
41 OnReservedFunction OnReserved) {
42 std::error_code EC;
43 auto MB = sys::Memory::allocateMappedMemory(
44 NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
45
46 if (EC)
47 return OnReserved(errorCodeToError(EC));
48
49 {
50 std::lock_guard<std::mutex> Lock(Mutex);
51 Reservations[MB.base()].Size = MB.allocatedSize();
52 }
53
54 OnReserved(
55 ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize()));
56 }
57
prepare(ExecutorAddr Addr,size_t ContentSize)58 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
59 return Addr.toPtr<char *>();
60 }
61
initialize(MemoryMapper::AllocInfo & AI,OnInitializedFunction OnInitialized)62 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
63 OnInitializedFunction OnInitialized) {
64 ExecutorAddr MinAddr(~0ULL);
65 ExecutorAddr MaxAddr(0);
66
67 // FIXME: Release finalize lifetime segments.
68 for (auto &Segment : AI.Segments) {
69 auto Base = AI.MappingBase + Segment.Offset;
70 auto Size = Segment.ContentSize + Segment.ZeroFillSize;
71
72 if (Base < MinAddr)
73 MinAddr = Base;
74
75 if (Base + Size > MaxAddr)
76 MaxAddr = Base + Size;
77
78 std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0,
79 Segment.ZeroFillSize);
80
81 if (auto EC = sys::Memory::protectMappedMemory(
82 {Base.toPtr<void *>(), Size},
83 toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) {
84 return OnInitialized(errorCodeToError(EC));
85 }
86 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
87 sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size);
88 }
89
90 auto DeinitializeActions = shared::runFinalizeActions(AI.Actions);
91 if (!DeinitializeActions)
92 return OnInitialized(DeinitializeActions.takeError());
93
94 {
95 std::lock_guard<std::mutex> Lock(Mutex);
96
97 // This is the maximum range whose permission have been possibly modified
98 Allocations[MinAddr].Size = MaxAddr - MinAddr;
99 Allocations[MinAddr].DeinitializationActions =
100 std::move(*DeinitializeActions);
101 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr);
102 }
103
104 OnInitialized(MinAddr);
105 }
106
deinitialize(ArrayRef<ExecutorAddr> Bases,MemoryMapper::OnDeinitializedFunction OnDeinitialized)107 void InProcessMemoryMapper::deinitialize(
108 ArrayRef<ExecutorAddr> Bases,
109 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
110 Error AllErr = Error::success();
111
112 {
113 std::lock_guard<std::mutex> Lock(Mutex);
114
115 for (auto Base : llvm::reverse(Bases)) {
116
117 if (Error Err = shared::runDeallocActions(
118 Allocations[Base].DeinitializationActions)) {
119 AllErr = joinErrors(std::move(AllErr), std::move(Err));
120 }
121
122 // Reset protections to read/write so the area can be reused
123 if (auto EC = sys::Memory::protectMappedMemory(
124 {Base.toPtr<void *>(), Allocations[Base].Size},
125 sys::Memory::ProtectionFlags::MF_READ |
126 sys::Memory::ProtectionFlags::MF_WRITE)) {
127 AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC));
128 }
129
130 Allocations.erase(Base);
131 }
132 }
133
134 OnDeinitialized(std::move(AllErr));
135 }
136
release(ArrayRef<ExecutorAddr> Bases,OnReleasedFunction OnReleased)137 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
138 OnReleasedFunction OnReleased) {
139 Error Err = Error::success();
140
141 for (auto Base : Bases) {
142 std::vector<ExecutorAddr> AllocAddrs;
143 size_t Size;
144 {
145 std::lock_guard<std::mutex> Lock(Mutex);
146 auto &R = Reservations[Base.toPtr<void *>()];
147 Size = R.Size;
148 AllocAddrs.swap(R.Allocations);
149 }
150
151 // deinitialize sub allocations
152 std::promise<MSVCPError> P;
153 auto F = P.get_future();
154 deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
155 if (Error E = F.get()) {
156 Err = joinErrors(std::move(Err), std::move(E));
157 }
158
159 // free the memory
160 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
161
162 auto EC = sys::Memory::releaseMappedMemory(MB);
163 if (EC) {
164 Err = joinErrors(std::move(Err), errorCodeToError(EC));
165 }
166
167 std::lock_guard<std::mutex> Lock(Mutex);
168 Reservations.erase(Base.toPtr<void *>());
169 }
170
171 OnReleased(std::move(Err));
172 }
173
~InProcessMemoryMapper()174 InProcessMemoryMapper::~InProcessMemoryMapper() {
175 std::vector<ExecutorAddr> ReservationAddrs;
176 {
177 std::lock_guard<std::mutex> Lock(Mutex);
178
179 ReservationAddrs.reserve(Reservations.size());
180 for (const auto &R : Reservations) {
181 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
182 }
183 }
184
185 std::promise<MSVCPError> P;
186 auto F = P.get_future();
187 release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
188 cantFail(F.get());
189 }
190
191 // SharedMemoryMapper
192
SharedMemoryMapper(ExecutorProcessControl & EPC,SymbolAddrs SAs,size_t PageSize)193 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC,
194 SymbolAddrs SAs, size_t PageSize)
195 : EPC(EPC), SAs(SAs), PageSize(PageSize) {
196 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
197 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
198 #endif
199 }
200
201 Expected<std::unique_ptr<SharedMemoryMapper>>
Create(ExecutorProcessControl & EPC,SymbolAddrs SAs)202 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) {
203 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
204 auto PageSize = sys::Process::getPageSize();
205 if (!PageSize)
206 return PageSize.takeError();
207
208 return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize);
209 #else
210 return make_error<StringError>(
211 "SharedMemoryMapper is not supported on this platform yet",
212 inconvertibleErrorCode());
213 #endif
214 }
215
reserve(size_t NumBytes,OnReservedFunction OnReserved)216 void SharedMemoryMapper::reserve(size_t NumBytes,
217 OnReservedFunction OnReserved) {
218 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
219
220 EPC.callSPSWrapperAsync<
221 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
222 SAs.Reserve,
223 [this, NumBytes, OnReserved = std::move(OnReserved)](
224 Error SerializationErr,
225 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
226 if (SerializationErr) {
227 cantFail(Result.takeError());
228 return OnReserved(std::move(SerializationErr));
229 }
230
231 if (!Result)
232 return OnReserved(Result.takeError());
233
234 ExecutorAddr RemoteAddr;
235 std::string SharedMemoryName;
236 std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result);
237
238 void *LocalAddr = nullptr;
239
240 #if defined(LLVM_ON_UNIX)
241
242 int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700);
243 if (SharedMemoryFile < 0) {
244 return OnReserved(errorCodeToError(
245 std::error_code(errno, std::generic_category())));
246 }
247
248 // this prevents other processes from accessing it by name
249 shm_unlink(SharedMemoryName.c_str());
250
251 LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
252 SharedMemoryFile, 0);
253 if (LocalAddr == MAP_FAILED) {
254 return OnReserved(errorCodeToError(
255 std::error_code(errno, std::generic_category())));
256 }
257
258 close(SharedMemoryFile);
259
260 #elif defined(_WIN32)
261
262 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
263 SharedMemoryName.end());
264 HANDLE SharedMemoryFile = OpenFileMappingW(
265 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
266 if (!SharedMemoryFile)
267 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
268
269 LocalAddr =
270 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
271 if (!LocalAddr) {
272 CloseHandle(SharedMemoryFile);
273 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
274 }
275
276 CloseHandle(SharedMemoryFile);
277
278 #endif
279 {
280 std::lock_guard<std::mutex> Lock(Mutex);
281 Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}});
282 }
283
284 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
285 },
286 SAs.Instance, static_cast<uint64_t>(NumBytes));
287
288 #else
289 OnReserved(make_error<StringError>(
290 "SharedMemoryMapper is not supported on this platform yet",
291 inconvertibleErrorCode()));
292 #endif
293 }
294
prepare(ExecutorAddr Addr,size_t ContentSize)295 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
296 auto R = Reservations.upper_bound(Addr);
297 assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
298 R--;
299
300 ExecutorAddrDiff Offset = Addr - R->first;
301
302 return static_cast<char *>(R->second.LocalAddr) + Offset;
303 }
304
initialize(MemoryMapper::AllocInfo & AI,OnInitializedFunction OnInitialized)305 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
306 OnInitializedFunction OnInitialized) {
307 auto Reservation = Reservations.upper_bound(AI.MappingBase);
308 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
309 Reservation--;
310
311 auto AllocationOffset = AI.MappingBase - Reservation->first;
312
313 tpctypes::SharedMemoryFinalizeRequest FR;
314
315 AI.Actions.swap(FR.Actions);
316
317 FR.Segments.reserve(AI.Segments.size());
318
319 for (auto Segment : AI.Segments) {
320 char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
321 AllocationOffset + Segment.Offset;
322 std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize);
323
324 tpctypes::SharedMemorySegFinalizeRequest SegReq;
325 SegReq.AG = Segment.AG;
326 SegReq.Addr = AI.MappingBase + Segment.Offset;
327 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
328
329 FR.Segments.push_back(SegReq);
330 }
331
332 EPC.callSPSWrapperAsync<
333 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
334 SAs.Initialize,
335 [OnInitialized = std::move(OnInitialized)](
336 Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
337 if (SerializationErr) {
338 cantFail(Result.takeError());
339 return OnInitialized(std::move(SerializationErr));
340 }
341
342 OnInitialized(std::move(Result));
343 },
344 SAs.Instance, Reservation->first, std::move(FR));
345 }
346
deinitialize(ArrayRef<ExecutorAddr> Allocations,MemoryMapper::OnDeinitializedFunction OnDeinitialized)347 void SharedMemoryMapper::deinitialize(
348 ArrayRef<ExecutorAddr> Allocations,
349 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
350 EPC.callSPSWrapperAsync<
351 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
352 SAs.Deinitialize,
353 [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
354 Error Result) mutable {
355 if (SerializationErr) {
356 cantFail(std::move(Result));
357 return OnDeinitialized(std::move(SerializationErr));
358 }
359
360 OnDeinitialized(std::move(Result));
361 },
362 SAs.Instance, Allocations);
363 }
364
release(ArrayRef<ExecutorAddr> Bases,OnReleasedFunction OnReleased)365 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
366 OnReleasedFunction OnReleased) {
367 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
368 Error Err = Error::success();
369
370 {
371 std::lock_guard<std::mutex> Lock(Mutex);
372
373 for (auto Base : Bases) {
374
375 #if defined(LLVM_ON_UNIX)
376
377 if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
378 Err = joinErrors(std::move(Err), errorCodeToError(std::error_code(
379 errno, std::generic_category())));
380
381 #elif defined(_WIN32)
382
383 if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
384 Err = joinErrors(std::move(Err),
385 errorCodeToError(mapWindowsError(GetLastError())));
386
387 #endif
388
389 Reservations.erase(Base);
390 }
391 }
392
393 EPC.callSPSWrapperAsync<
394 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
395 SAs.Release,
396 [OnReleased = std::move(OnReleased),
397 Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
398 if (SerializationErr) {
399 cantFail(std::move(Result));
400 return OnReleased(
401 joinErrors(std::move(Err), std::move(SerializationErr)));
402 }
403
404 return OnReleased(joinErrors(std::move(Err), std::move(Result)));
405 },
406 SAs.Instance, Bases);
407 #else
408 OnReleased(make_error<StringError>(
409 "SharedMemoryMapper is not supported on this platform yet",
410 inconvertibleErrorCode()));
411 #endif
412 }
413
~SharedMemoryMapper()414 SharedMemoryMapper::~SharedMemoryMapper() {
415 std::lock_guard<std::mutex> Lock(Mutex);
416 for (const auto &R : Reservations) {
417
418 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
419
420 munmap(R.second.LocalAddr, R.second.Size);
421
422 #elif defined(_WIN32)
423
424 UnmapViewOfFile(R.second.LocalAddr);
425
426 #else
427
428 (void)R;
429
430 #endif
431 }
432 }
433
434 } // namespace orc
435
436 } // namespace llvm
437