1 /*
2 * Copyright 2019 Google
3 * SPDX-License-Identifier: MIT
4 */
5
6 #include "goldfish_address_space.h"
7
8 #include <errno.h>
9 #include <fcntl.h>
10 #include <linux/ioctl.h>
11 #include <linux/types.h>
12 #include <sys/ioctl.h>
13 #include <sys/mman.h>
14 #include <sys/stat.h>
15 #include <sys/types.h>
16 #include <unistd.h>
17
18 #include <cstdlib>
19 #include <cstring>
20 #include <memory>
21
22 #include "util/log.h"
23
24 // See virgl_hw.h and p_defines.h
25 #define VIRGL_FORMAT_R8_UNORM 64
26 #define VIRGL_BIND_CUSTOM (1 << 17)
27 #define PIPE_BUFFER 0
28
29 #ifdef PAGE_SIZE
30 constexpr size_t kPageSize = PAGE_SIZE;
31 #else
32 static const size_t kPageSize = getpagesize();
33 #endif
34
35 namespace {
36
37 struct goldfish_address_space_allocate_block {
38 __u64 size;
39 __u64 offset;
40 __u64 phys_addr;
41 };
42
43 struct goldfish_address_space_claim_shared {
44 __u64 offset;
45 __u64 size;
46 };
47
48 #define GOLDFISH_ADDRESS_SPACE_IOCTL_MAGIC 'G'
49 #define GOLDFISH_ADDRESS_SPACE_IOCTL_OP(OP, T) _IOWR(GOLDFISH_ADDRESS_SPACE_IOCTL_MAGIC, OP, T)
50 #define GOLDFISH_ADDRESS_SPACE_IOCTL_ALLOCATE_BLOCK GOLDFISH_ADDRESS_SPACE_IOCTL_OP(10, struct goldfish_address_space_allocate_block)
51 #define GOLDFISH_ADDRESS_SPACE_IOCTL_DEALLOCATE_BLOCK GOLDFISH_ADDRESS_SPACE_IOCTL_OP(11, __u64)
52 #define GOLDFISH_ADDRESS_SPACE_IOCTL_PING GOLDFISH_ADDRESS_SPACE_IOCTL_OP(12, struct address_space_ping)
53 #define GOLDFISH_ADDRESS_SPACE_IOCTL_CLAIM_SHARED GOLDFISH_ADDRESS_SPACE_IOCTL_OP(13, struct goldfish_address_space_claim_shared)
54 #define GOLDFISH_ADDRESS_SPACE_IOCTL_UNCLAIM_SHARED GOLDFISH_ADDRESS_SPACE_IOCTL_OP(14, __u64)
55
56 const char GOLDFISH_ADDRESS_SPACE_DEVICE_NAME[] = "/dev/goldfish_address_space";
57
58 const int HOST_MEMORY_ALLOCATOR_COMMAND_ALLOCATE_ID = 1;
59 const int HOST_MEMORY_ALLOCATOR_COMMAND_UNALLOCATE_ID = 2;
60
create_address_space_fd()61 int create_address_space_fd()
62 {
63 return ::open(GOLDFISH_ADDRESS_SPACE_DEVICE_NAME, O_RDWR);
64 }
65
ioctl_allocate(int fd,struct goldfish_address_space_allocate_block * request)66 long ioctl_allocate(int fd, struct goldfish_address_space_allocate_block *request)
67 {
68 return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_ALLOCATE_BLOCK, request);
69 }
70
ioctl_deallocate(int fd,uint64_t offset)71 long ioctl_deallocate(int fd, uint64_t offset)
72 {
73 return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_DEALLOCATE_BLOCK, &offset);
74 }
75
ioctl_ping(int fd,struct address_space_ping * request)76 long ioctl_ping(int fd, struct address_space_ping *request)
77 {
78 return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_PING, request);
79 }
80
set_address_space_subdevice_type(int fd,uint64_t type)81 long set_address_space_subdevice_type(int fd, uint64_t type)
82 {
83 struct address_space_ping request;
84 ::memset(&request, 0, sizeof(request));
85 request.resourceId = sizeof(request);
86 request.metadata = type;
87
88 long ret = ioctl_ping(fd, &request);
89 if (ret) {
90 return ret;
91 }
92
93 return request.metadata;
94 }
95
ioctl_claim_shared(int fd,struct goldfish_address_space_claim_shared * request)96 long ioctl_claim_shared(int fd, struct goldfish_address_space_claim_shared *request)
97 {
98 return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_CLAIM_SHARED, request);
99 }
100
ioctl_unclaim_shared(int fd,uint64_t offset)101 long ioctl_unclaim_shared(int fd, uint64_t offset)
102 {
103 return ::ioctl(fd, GOLDFISH_ADDRESS_SPACE_IOCTL_UNCLAIM_SHARED, &offset);
104 }
105
106 } // namespace
107
GoldfishAddressSpaceBlockProvider(GoldfishAddressSpaceSubdeviceType subdevice)108 GoldfishAddressSpaceBlockProvider::GoldfishAddressSpaceBlockProvider(GoldfishAddressSpaceSubdeviceType subdevice)
109 : m_handle(create_address_space_fd())
110 {
111 if ((subdevice != GoldfishAddressSpaceSubdeviceType::NoSubdevice) && is_opened()) {
112 const long ret = set_address_space_subdevice_type(m_handle, subdevice);
113 if (ret != 0 && ret != subdevice) { // TODO: retire the 'ret != subdevice' check
114 mesa_loge("%s: set_address_space_subdevice_type failed for device_type=%lu, ret=%ld",
115 __func__, static_cast<unsigned long>(subdevice), ret);
116 close();
117 }
118 }
119 }
120
~GoldfishAddressSpaceBlockProvider()121 GoldfishAddressSpaceBlockProvider::~GoldfishAddressSpaceBlockProvider()
122 {
123 if (is_opened()) {
124 ::close(m_handle);
125 }
126 }
127
is_opened() const128 bool GoldfishAddressSpaceBlockProvider::is_opened() const
129 {
130 return m_handle >= 0;
131 }
132
close()133 void GoldfishAddressSpaceBlockProvider::close()
134 {
135 if (is_opened()) {
136 ::close(m_handle);
137 m_handle = -1;
138 }
139 }
140
release()141 address_space_handle_t GoldfishAddressSpaceBlockProvider::release()
142 {
143 address_space_handle_t handle = m_handle;
144 m_handle = -1;
145 return handle;
146 }
147
closeHandle(address_space_handle_t handle)148 void GoldfishAddressSpaceBlockProvider::closeHandle(address_space_handle_t handle)
149 {
150 ::close(handle);
151 }
152
GoldfishAddressSpaceBlock()153 GoldfishAddressSpaceBlock::GoldfishAddressSpaceBlock()
154 : m_handle(-1)
155 , m_mmaped_ptr(NULL)
156 , m_phys_addr(0)
157 , m_host_addr(0)
158 , m_offset(0)
159 , m_size(0) {}
160
~GoldfishAddressSpaceBlock()161 GoldfishAddressSpaceBlock::~GoldfishAddressSpaceBlock()
162 {
163 destroy();
164 }
165
operator =(const GoldfishAddressSpaceBlock & rhs)166 GoldfishAddressSpaceBlock &GoldfishAddressSpaceBlock::operator=(const GoldfishAddressSpaceBlock &rhs)
167 {
168 m_mmaped_ptr = rhs.m_mmaped_ptr;
169 m_phys_addr = rhs.m_phys_addr;
170 m_host_addr = rhs.m_host_addr;
171 m_offset = rhs.m_offset;
172 m_size = rhs.m_size;
173 m_handle = rhs.m_handle;
174
175 return *this;
176 }
177
allocate(GoldfishAddressSpaceBlockProvider * provider,size_t size)178 bool GoldfishAddressSpaceBlock::allocate(GoldfishAddressSpaceBlockProvider *provider, size_t size)
179 {
180 destroy();
181
182 if (!provider->is_opened()) {
183 return false;
184 }
185
186 struct goldfish_address_space_allocate_block request;
187 ::memset(&request, 0, sizeof(request));
188 request.size = size;
189
190 long res = ioctl_allocate(provider->m_handle, &request);
191 if (res) {
192 return false;
193 } else {
194 m_phys_addr = request.phys_addr;
195 m_offset = request.offset;
196 m_size = request.size;
197 m_handle = provider->m_handle;
198 m_is_shared_mapping = false;
199
200 return true;
201 }
202 }
203
claimShared(GoldfishAddressSpaceBlockProvider * provider,uint64_t offset,uint64_t size)204 bool GoldfishAddressSpaceBlock::claimShared(GoldfishAddressSpaceBlockProvider *provider, uint64_t offset, uint64_t size)
205 {
206 destroy();
207
208 if (!provider->is_opened()) {
209 return false;
210 }
211
212 struct goldfish_address_space_claim_shared request;
213 request.offset = offset;
214 request.size = size;
215 long res = ioctl_claim_shared(provider->m_handle, &request);
216
217 if (res) {
218 return false;
219 }
220
221 m_offset = offset;
222 m_size = size;
223 m_handle = provider->m_handle;
224 m_is_shared_mapping = true;
225
226 return true;
227 }
228
physAddr() const229 uint64_t GoldfishAddressSpaceBlock::physAddr() const
230 {
231 return m_phys_addr;
232 }
233
hostAddr() const234 uint64_t GoldfishAddressSpaceBlock::hostAddr() const
235 {
236 return m_host_addr;
237 }
238
mmap(uint64_t host_addr)239 void *GoldfishAddressSpaceBlock::mmap(uint64_t host_addr)
240 {
241 if (m_size == 0) {
242 mesa_loge("%s: called with zero size\n", __func__);
243 return NULL;
244 }
245 if (m_mmaped_ptr) {
246 mesa_loge("'mmap' called for an already mmaped address block");
247 ::abort();
248 }
249
250 void *result;
251 const int res = memoryMap(NULL, m_size, m_handle, m_offset, &result);
252 if (res) {
253 mesa_loge(
254 "%s: host memory map failed with size 0x%llx "
255 "off 0x%llx errno %d\n",
256 __func__, (unsigned long long)m_size, (unsigned long long)m_offset, res);
257 return NULL;
258 } else {
259 m_mmaped_ptr = result;
260 m_host_addr = host_addr;
261 return guestPtr();
262 }
263 }
264
guestPtr() const265 void *GoldfishAddressSpaceBlock::guestPtr() const
266 {
267 return reinterpret_cast<char *>(m_mmaped_ptr) + (m_host_addr & (kPageSize - 1));
268 }
269
destroy()270 void GoldfishAddressSpaceBlock::destroy()
271 {
272 if (m_mmaped_ptr && m_size) {
273 memoryUnmap(m_mmaped_ptr, m_size);
274 m_mmaped_ptr = NULL;
275 }
276
277 if (m_size) {
278 long res = -EINVAL;
279
280 if (m_is_shared_mapping) {
281 res = ioctl_unclaim_shared(m_handle, m_offset);
282 if (res) {
283 mesa_loge("ioctl_unclaim_shared failed, res=%ld", res);
284 ::abort();
285 }
286 } else {
287 res = ioctl_deallocate(m_handle, m_offset);
288 if (res) {
289 mesa_loge("ioctl_deallocate failed, res=%ld", res);
290 ::abort();
291 }
292 }
293
294 m_is_shared_mapping = false;
295
296 m_phys_addr = 0;
297 m_host_addr = 0;
298 m_offset = 0;
299 m_size = 0;
300 }
301 }
302
release()303 void GoldfishAddressSpaceBlock::release()
304 {
305 m_handle = -1;
306 m_mmaped_ptr = NULL;
307 m_phys_addr = 0;
308 m_host_addr = 0;
309 m_offset = 0;
310 m_size = 0;
311 }
312
memoryMap(void * addr,size_t len,address_space_handle_t fd,uint64_t off,void ** dst)313 int GoldfishAddressSpaceBlock::memoryMap(void *addr,
314 size_t len,
315 address_space_handle_t fd,
316 uint64_t off,
317 void** dst) {
318 void* ptr = ::mmap64(addr, len, PROT_WRITE, MAP_SHARED, fd, off);
319 if (MAP_FAILED == ptr) {
320 return errno;
321 } else {
322 *dst = ptr;
323 return 0;
324 }
325 }
326
memoryUnmap(void * ptr,size_t size)327 void GoldfishAddressSpaceBlock::memoryUnmap(void *ptr, size_t size)
328 {
329 ::munmap(ptr, size);
330 }
331
GoldfishAddressSpaceHostMemoryAllocator(bool useSharedSlots)332 GoldfishAddressSpaceHostMemoryAllocator::GoldfishAddressSpaceHostMemoryAllocator(bool useSharedSlots)
333 : m_provider(useSharedSlots
334 ? GoldfishAddressSpaceSubdeviceType::SharedSlotsHostMemoryAllocator
335 : GoldfishAddressSpaceSubdeviceType::HostMemoryAllocator),
336 m_useSharedSlots(useSharedSlots)
337 {}
338
is_opened() const339 bool GoldfishAddressSpaceHostMemoryAllocator::is_opened() const { return m_provider.is_opened(); }
340
hostMalloc(GoldfishAddressSpaceBlock * block,size_t size)341 long GoldfishAddressSpaceHostMemoryAllocator::hostMalloc(GoldfishAddressSpaceBlock *block, size_t size)
342 {
343 if (size == 0) {
344 return -EINVAL;
345 }
346 if (block->size() > 0) {
347 return -EINVAL;
348 }
349 if (!m_provider.is_opened()) {
350 return -ENODEV;
351 }
352
353 struct address_space_ping request;
354 if (m_useSharedSlots) {
355 // shared memory slots are supported
356 ::memset(&request, 0, sizeof(request));
357 request.resourceId = sizeof(request);
358 request.size = size;
359 request.metadata = HOST_MEMORY_ALLOCATOR_COMMAND_ALLOCATE_ID;
360
361 long ret = ioctl_ping(m_provider.m_handle, &request);
362 if (ret) {
363 return ret;
364 }
365 ret = static_cast<long>(request.metadata);
366 if (ret) {
367 return ret;
368 }
369
370 block->claimShared(&m_provider, request.offset, request.size);
371 } else {
372 // shared memory slots are not supported
373 if (!block->allocate(&m_provider, size)) {
374 return -ENOMEM;
375 }
376
377 ::memset(&request, 0, sizeof(request));
378 request.resourceId = sizeof(request);
379 request.offset = block->offset();
380 request.size = block->size();
381 request.metadata = HOST_MEMORY_ALLOCATOR_COMMAND_ALLOCATE_ID;
382
383 long ret = ioctl_ping(m_provider.m_handle, &request);
384 if (ret) {
385 return ret;
386 }
387 ret = static_cast<long>(request.metadata);
388 if (ret) {
389 return ret;
390 }
391 }
392
393 block->mmap(0);
394 return 0;
395 }
396
hostFree(GoldfishAddressSpaceBlock * block)397 void GoldfishAddressSpaceHostMemoryAllocator::hostFree(GoldfishAddressSpaceBlock *block)
398 {
399 if (block->size() == 0) {
400 return;
401 }
402
403 if (!m_provider.is_opened()) {
404 mesa_loge("%s: device is not available", __func__);
405 ::abort();
406 }
407
408 if (block->guestPtr()) {
409 struct address_space_ping request;
410 ::memset(&request, 0, sizeof(request));
411 request.resourceId = sizeof(request);
412 request.offset = block->offset();
413 request.metadata = HOST_MEMORY_ALLOCATOR_COMMAND_UNALLOCATE_ID;
414
415 const long ret = ioctl_ping(m_provider.m_handle, &request);
416 if (ret) {
417 mesa_loge("%s: ioctl_ping failed, ret=%ld", __func__, ret);
418 ::abort();
419 }
420 }
421
422 block->replace(NULL);
423 }
424
goldfish_address_space_open()425 address_space_handle_t goldfish_address_space_open() {
426 return ::open(GOLDFISH_ADDRESS_SPACE_DEVICE_NAME, O_RDWR);
427 }
428
goldfish_address_space_close(address_space_handle_t handle)429 void goldfish_address_space_close(address_space_handle_t handle) {
430 ::close(handle);
431 }
432
goldfish_address_space_allocate(address_space_handle_t handle,size_t size,uint64_t * phys_addr,uint64_t * offset)433 bool goldfish_address_space_allocate(
434 address_space_handle_t handle,
435 size_t size, uint64_t* phys_addr, uint64_t* offset) {
436
437 struct goldfish_address_space_allocate_block request;
438 ::memset(&request, 0, sizeof(request));
439 request.size = size;
440
441 long res = ioctl_allocate(handle, &request);
442
443 if (res) return false;
444
445 *phys_addr = request.phys_addr;
446 *offset = request.offset;
447 return true;
448 }
449
goldfish_address_space_free(address_space_handle_t handle,uint64_t offset)450 bool goldfish_address_space_free(
451 address_space_handle_t handle, uint64_t offset) {
452
453 long res = ioctl_deallocate(handle, offset);
454
455 if (res) {
456 mesa_loge("ioctl_deallocate failed, res=%ld", res);
457 ::abort();
458 }
459
460 return true;
461 }
462
goldfish_address_space_claim_shared(address_space_handle_t handle,uint64_t offset,uint64_t size)463 bool goldfish_address_space_claim_shared(
464 address_space_handle_t handle, uint64_t offset, uint64_t size) {
465
466 struct goldfish_address_space_claim_shared request;
467 request.offset = offset;
468 request.size = size;
469 long res = ioctl_claim_shared(handle, &request);
470
471 if (res) return false;
472
473 return true;
474 }
475
goldfish_address_space_unclaim_shared(address_space_handle_t handle,uint64_t offset)476 bool goldfish_address_space_unclaim_shared(
477 address_space_handle_t handle, uint64_t offset) {
478 long res = ioctl_unclaim_shared(handle, offset);
479 if (res) {
480 mesa_loge("ioctl_unclaim_shared failed, res=%ld", res);
481 ::abort();
482 }
483
484 return true;
485 }
486
487 // pgoff is the offset into the page to return in the result
goldfish_address_space_map(address_space_handle_t handle,uint64_t offset,uint64_t size,uint64_t pgoff)488 void* goldfish_address_space_map(
489 address_space_handle_t handle,
490 uint64_t offset, uint64_t size,
491 uint64_t pgoff) {
492
493 void* res = ::mmap64(0, size, PROT_WRITE, MAP_SHARED, handle, offset);
494
495 if (res == MAP_FAILED) {
496 mesa_loge("%s: failed to map. errno: %d\n", __func__, errno);
497 return 0;
498 }
499
500 return (void*)(((char*)res) + (uintptr_t)(pgoff & (kPageSize - 1)));
501 }
502
goldfish_address_space_unmap(void * ptr,uint64_t size)503 void goldfish_address_space_unmap(void* ptr, uint64_t size) {
504 void* pagePtr = (void*)(((uintptr_t)ptr) & ~(kPageSize - 1));
505 ::munmap(pagePtr, size);
506 }
507
goldfish_address_space_set_subdevice_type(address_space_handle_t handle,GoldfishAddressSpaceSubdeviceType type,address_space_handle_t * handle_out)508 bool goldfish_address_space_set_subdevice_type(
509 address_space_handle_t handle, GoldfishAddressSpaceSubdeviceType type,
510 address_space_handle_t* handle_out) {
511 struct address_space_ping request;
512 request.metadata = (uint64_t)type;
513 *handle_out = handle;
514 return goldfish_address_space_ping(handle, &request);
515 }
516
goldfish_address_space_ping(address_space_handle_t handle,struct address_space_ping * ping)517 bool goldfish_address_space_ping(
518 address_space_handle_t handle,
519 struct address_space_ping* ping) {
520 long res = ioctl_ping(handle, ping);
521
522 if (res) {
523 mesa_loge("%s: ping failed: errno: %d\n", __func__, errno);
524 return false;
525 }
526
527 return true;
528 }
529
replace(GoldfishAddressSpaceBlock * other)530 void GoldfishAddressSpaceBlock::replace(GoldfishAddressSpaceBlock *other)
531 {
532 destroy();
533
534 if (other) {
535 *this = *other;
536 *other = GoldfishAddressSpaceBlock();
537 }
538 }
539