1 /*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <sys/types.h>
27 #include <sys/stat.h>
28 #include <fcntl.h>
29 #if HAVE_ALLOCA_H
30 # include <alloca.h>
31 #endif
32
33 #include "CUnit/Basic.h"
34
35 #include "amdgpu_test.h"
36 #include "amdgpu_drm.h"
37 #include "amdgpu_internal.h"
38 #include "xf86drm.h"
39 #include <pthread.h>
40
41 #define GFX_COMPUTE_NOP 0xffff1000
42
43 static amdgpu_device_handle device_handle;
44 static uint32_t major_version;
45 static uint32_t minor_version;
46 static char *sysfs_remove = NULL;
47 static bool do_cs;
48
suite_hotunplug_tests_enable(void)49 CU_BOOL suite_hotunplug_tests_enable(void)
50 {
51 CU_BOOL enable = CU_TRUE;
52 drmDevicePtr device;
53
54 if (drmGetDevice2(drm_amdgpu[0], DRM_DEVICE_GET_PCI_REVISION, &device)) {
55 printf("\n\nGPU Failed to get DRM device PCI info!\n");
56 return CU_FALSE;
57 }
58
59 if (device->bustype != DRM_BUS_PCI) {
60 printf("\n\nGPU device is not on PCI bus!\n");
61 amdgpu_device_deinitialize(device_handle);
62 return CU_FALSE;
63 }
64
65 if (amdgpu_device_initialize(drm_amdgpu[0], &major_version,
66 &minor_version, &device_handle))
67 return CU_FALSE;
68
69 /* Latest tested amdgpu version to work with all the tests */
70 if (minor_version < 46)
71 enable = false;
72
73 /* skip hotplug test on APUs */
74 if(device_handle->dev_info.ids_flags & AMDGPU_IDS_FLAGS_FUSION)
75 enable = false;
76
77 if (amdgpu_device_deinitialize(device_handle))
78 return CU_FALSE;
79
80 return enable;
81 }
82
suite_hotunplug_tests_init(void)83 int suite_hotunplug_tests_init(void)
84 {
85 /* We need to open/close device at each test manually */
86 amdgpu_close_devices();
87
88 return CUE_SUCCESS;
89 }
90
suite_hotunplug_tests_clean(void)91 int suite_hotunplug_tests_clean(void)
92 {
93
94
95 return CUE_SUCCESS;
96 }
97
amdgpu_hotunplug_trigger(const char * pathname)98 static int amdgpu_hotunplug_trigger(const char *pathname)
99 {
100 int fd, len;
101
102 fd = open(pathname, O_WRONLY);
103 if (fd < 0)
104 return -errno;
105
106 len = write(fd, "1", 1);
107 close(fd);
108
109 return len;
110 }
111
amdgpu_hotunplug_setup_test()112 static int amdgpu_hotunplug_setup_test()
113 {
114 int r;
115 char *tmp_str;
116
117 if (amdgpu_open_device_on_test_index(open_render_node) < 0) {
118 printf("\n\n Failed to reopen device file!\n");
119 return CUE_SINIT_FAILED;
120
121
122
123 }
124
125 r = amdgpu_device_initialize(drm_amdgpu[0], &major_version,
126 &minor_version, &device_handle);
127
128 if (r) {
129 if ((r == -EACCES) && (errno == EACCES))
130 printf("\n\nError:%s. "
131 "Hint:Try to run this test program as root.",
132 strerror(errno));
133 return CUE_SINIT_FAILED;
134 }
135
136 tmp_str = amdgpu_get_device_from_fd(drm_amdgpu[0]);
137 if (!tmp_str){
138 printf("\n\n Device path not found!\n");
139 return CUE_SINIT_FAILED;
140 }
141
142 sysfs_remove = realloc(tmp_str, strlen(tmp_str) * 2);
143 strcat(sysfs_remove, "/remove");
144
145 return 0;
146 }
147
amdgpu_hotunplug_teardown_test()148 static int amdgpu_hotunplug_teardown_test()
149 {
150 if (amdgpu_device_deinitialize(device_handle))
151 return CUE_SCLEAN_FAILED;
152
153 amdgpu_close_devices();
154
155 if (sysfs_remove)
156 free(sysfs_remove);
157
158 return 0;
159 }
160
amdgpu_hotunplug_remove()161 static inline int amdgpu_hotunplug_remove()
162 {
163 return amdgpu_hotunplug_trigger(sysfs_remove);
164 }
165
amdgpu_hotunplug_rescan()166 static inline int amdgpu_hotunplug_rescan()
167 {
168 return amdgpu_hotunplug_trigger("/sys/bus/pci/rescan");
169 }
170
amdgpu_cs_sync(amdgpu_context_handle context,unsigned int ip_type,int ring,unsigned int seqno)171 static int amdgpu_cs_sync(amdgpu_context_handle context,
172 unsigned int ip_type,
173 int ring,
174 unsigned int seqno)
175 {
176 struct amdgpu_cs_fence fence = {
177 .context = context,
178 .ip_type = ip_type,
179 .ring = ring,
180 .fence = seqno,
181 };
182 uint32_t expired;
183
184 return amdgpu_cs_query_fence_status(&fence,
185 AMDGPU_TIMEOUT_INFINITE,
186 0, &expired);
187 }
188
amdgpu_nop_cs()189 static void *amdgpu_nop_cs()
190 {
191 amdgpu_bo_handle ib_result_handle;
192 void *ib_result_cpu;
193 uint64_t ib_result_mc_address;
194 uint32_t *ptr;
195 int i, r;
196 amdgpu_bo_list_handle bo_list;
197 amdgpu_va_handle va_handle;
198 amdgpu_context_handle context;
199 struct amdgpu_cs_request ibs_request;
200 struct amdgpu_cs_ib_info ib_info;
201
202 r = amdgpu_cs_ctx_create(device_handle, &context);
203 CU_ASSERT_EQUAL(r, 0);
204
205 r = amdgpu_bo_alloc_and_map(device_handle, 4096, 4096,
206 AMDGPU_GEM_DOMAIN_GTT, 0,
207 &ib_result_handle, &ib_result_cpu,
208 &ib_result_mc_address, &va_handle);
209 CU_ASSERT_EQUAL(r, 0);
210
211 ptr = ib_result_cpu;
212 for (i = 0; i < 16; ++i)
213 ptr[i] = GFX_COMPUTE_NOP;
214
215 r = amdgpu_bo_list_create(device_handle, 1, &ib_result_handle, NULL, &bo_list);
216 CU_ASSERT_EQUAL(r, 0);
217
218 memset(&ib_info, 0, sizeof(struct amdgpu_cs_ib_info));
219 ib_info.ib_mc_address = ib_result_mc_address;
220 ib_info.size = 16;
221
222 memset(&ibs_request, 0, sizeof(struct amdgpu_cs_request));
223 ibs_request.ip_type = AMDGPU_HW_IP_GFX;
224 ibs_request.ring = 0;
225 ibs_request.number_of_ibs = 1;
226 ibs_request.ibs = &ib_info;
227 ibs_request.resources = bo_list;
228
229 while (do_cs)
230 amdgpu_cs_submit(context, 0, &ibs_request, 1);
231
232 amdgpu_cs_sync(context, AMDGPU_HW_IP_GFX, 0, ibs_request.seq_no);
233 amdgpu_bo_list_destroy(bo_list);
234 amdgpu_bo_unmap_and_free(ib_result_handle, va_handle,
235 ib_result_mc_address, 4096);
236
237 amdgpu_cs_ctx_free(context);
238
239 return (void *)0;
240 }
241
amdgpu_create_cs_thread()242 static pthread_t* amdgpu_create_cs_thread()
243 {
244 int r;
245 pthread_t *thread = malloc(sizeof(*thread));
246 if (!thread)
247 return NULL;
248
249 do_cs = true;
250
251 r = pthread_create(thread, NULL, amdgpu_nop_cs, NULL);
252 CU_ASSERT_EQUAL(r, 0);
253
254 /* Give thread enough time to start*/
255 usleep(100000);
256 return thread;
257 }
258
amdgpu_destroy_cs_thread(pthread_t * thread)259 static void amdgpu_destroy_cs_thread(pthread_t *thread)
260 {
261 void *status;
262
263 do_cs = false;
264
265 pthread_join(*thread, &status);
266 CU_ASSERT_EQUAL(status, 0);
267
268 free(thread);
269 }
270
271
amdgpu_hotunplug_test(bool with_cs)272 static void amdgpu_hotunplug_test(bool with_cs)
273 {
274 int r;
275 pthread_t *thread = NULL;
276
277 r = amdgpu_hotunplug_setup_test();
278 CU_ASSERT_EQUAL(r , 0);
279
280 if (with_cs) {
281 thread = amdgpu_create_cs_thread();
282 CU_ASSERT_NOT_EQUAL(thread, NULL);
283 }
284
285 r = amdgpu_hotunplug_remove();
286 CU_ASSERT_EQUAL(r > 0, 1);
287
288 if (with_cs)
289 amdgpu_destroy_cs_thread(thread);
290
291 r = amdgpu_hotunplug_teardown_test();
292 CU_ASSERT_EQUAL(r , 0);
293
294 r = amdgpu_hotunplug_rescan();
295 CU_ASSERT_EQUAL(r > 0, 1);
296 }
297
amdgpu_hotunplug_simple(void)298 static void amdgpu_hotunplug_simple(void)
299 {
300 amdgpu_hotunplug_test(false);
301 }
302
amdgpu_hotunplug_with_cs(void)303 static void amdgpu_hotunplug_with_cs(void)
304 {
305 amdgpu_hotunplug_test(true);
306 }
307
amdgpu_hotunplug_with_exported_bo(void)308 static void amdgpu_hotunplug_with_exported_bo(void)
309 {
310 int r;
311 uint32_t dma_buf_fd;
312 unsigned int *ptr;
313 amdgpu_bo_handle bo_handle;
314
315 struct amdgpu_bo_alloc_request request = {
316 .alloc_size = 4096,
317 .phys_alignment = 4096,
318 .preferred_heap = AMDGPU_GEM_DOMAIN_GTT,
319 .flags = 0,
320 };
321
322 r = amdgpu_hotunplug_setup_test();
323 CU_ASSERT_EQUAL(r , 0);
324
325 amdgpu_bo_alloc(device_handle, &request, &bo_handle);
326 CU_ASSERT_EQUAL(r, 0);
327
328 r = amdgpu_bo_export(bo_handle, amdgpu_bo_handle_type_dma_buf_fd, &dma_buf_fd);
329 CU_ASSERT_EQUAL(r, 0);
330
331 ptr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, dma_buf_fd, 0);
332 CU_ASSERT_NOT_EQUAL(ptr, MAP_FAILED);
333
334 r = amdgpu_hotunplug_remove();
335 CU_ASSERT_EQUAL(r > 0, 1);
336
337 amdgpu_bo_free(bo_handle);
338
339 r = amdgpu_hotunplug_teardown_test();
340 CU_ASSERT_EQUAL(r , 0);
341
342 *ptr = 0xdeafbeef;
343
344 munmap(ptr, 4096);
345 close (dma_buf_fd);
346
347 r = amdgpu_hotunplug_rescan();
348 CU_ASSERT_EQUAL(r > 0, 1);
349 }
350
amdgpu_hotunplug_with_exported_fence(void)351 static void amdgpu_hotunplug_with_exported_fence(void)
352 {
353 amdgpu_bo_handle ib_result_handle;
354 void *ib_result_cpu;
355 uint64_t ib_result_mc_address;
356 uint32_t *ptr, sync_obj_handle, sync_obj_handle2;
357 int i, r;
358 amdgpu_bo_list_handle bo_list;
359 amdgpu_va_handle va_handle;
360 uint32_t major2, minor2;
361 amdgpu_device_handle device2;
362 amdgpu_context_handle context;
363 struct amdgpu_cs_request ibs_request;
364 struct amdgpu_cs_ib_info ib_info;
365 struct amdgpu_cs_fence fence_status = {0};
366 int shared_fd;
367
368 r = amdgpu_hotunplug_setup_test();
369 CU_ASSERT_EQUAL(r , 0);
370
371 r = amdgpu_device_initialize(drm_amdgpu[1], &major2, &minor2, &device2);
372 CU_ASSERT_EQUAL(r, 0);
373
374 r = amdgpu_cs_ctx_create(device_handle, &context);
375 CU_ASSERT_EQUAL(r, 0);
376
377 r = amdgpu_bo_alloc_and_map(device_handle, 4096, 4096,
378 AMDGPU_GEM_DOMAIN_GTT, 0,
379 &ib_result_handle, &ib_result_cpu,
380 &ib_result_mc_address, &va_handle);
381 CU_ASSERT_EQUAL(r, 0);
382
383 ptr = ib_result_cpu;
384 for (i = 0; i < 16; ++i)
385 ptr[i] = GFX_COMPUTE_NOP;
386
387 r = amdgpu_bo_list_create(device_handle, 1, &ib_result_handle, NULL, &bo_list);
388 CU_ASSERT_EQUAL(r, 0);
389
390 memset(&ib_info, 0, sizeof(struct amdgpu_cs_ib_info));
391 ib_info.ib_mc_address = ib_result_mc_address;
392 ib_info.size = 16;
393
394 memset(&ibs_request, 0, sizeof(struct amdgpu_cs_request));
395 ibs_request.ip_type = AMDGPU_HW_IP_GFX;
396 ibs_request.ring = 0;
397 ibs_request.number_of_ibs = 1;
398 ibs_request.ibs = &ib_info;
399 ibs_request.resources = bo_list;
400
401 CU_ASSERT_EQUAL(amdgpu_cs_submit(context, 0, &ibs_request, 1), 0);
402
403 fence_status.context = context;
404 fence_status.ip_type = AMDGPU_HW_IP_GFX;
405 fence_status.ip_instance = 0;
406 fence_status.fence = ibs_request.seq_no;
407
408 CU_ASSERT_EQUAL(amdgpu_cs_fence_to_handle(device_handle, &fence_status,
409 AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ,
410 &sync_obj_handle),
411 0);
412
413 CU_ASSERT_EQUAL(amdgpu_cs_export_syncobj(device_handle, sync_obj_handle, &shared_fd), 0);
414
415 CU_ASSERT_EQUAL(amdgpu_cs_import_syncobj(device2, shared_fd, &sync_obj_handle2), 0);
416
417 CU_ASSERT_EQUAL(amdgpu_cs_destroy_syncobj(device_handle, sync_obj_handle), 0);
418
419 CU_ASSERT_EQUAL(amdgpu_bo_list_destroy(bo_list), 0);
420 CU_ASSERT_EQUAL(amdgpu_bo_unmap_and_free(ib_result_handle, va_handle,
421 ib_result_mc_address, 4096), 0);
422 CU_ASSERT_EQUAL(amdgpu_cs_ctx_free(context), 0);
423
424 r = amdgpu_hotunplug_remove();
425 CU_ASSERT_EQUAL(r > 0, 1);
426
427 CU_ASSERT_EQUAL(amdgpu_cs_syncobj_wait(device2, &sync_obj_handle2, 1, 100000000, 0, NULL), 0);
428
429 CU_ASSERT_EQUAL(amdgpu_cs_destroy_syncobj(device2, sync_obj_handle2), 0);
430
431 amdgpu_device_deinitialize(device2);
432
433 r = amdgpu_hotunplug_teardown_test();
434 CU_ASSERT_EQUAL(r , 0);
435
436 r = amdgpu_hotunplug_rescan();
437 CU_ASSERT_EQUAL(r > 0, 1);
438 }
439
440
441 CU_TestInfo hotunplug_tests[] = {
442 { "Unplug card and rescan the bus to plug it back", amdgpu_hotunplug_simple },
443 { "Same as first test but with command submission", amdgpu_hotunplug_with_cs },
444 { "Unplug with exported bo", amdgpu_hotunplug_with_exported_bo },
445 { "Unplug with exported fence", amdgpu_hotunplug_with_exported_fence },
446 CU_TEST_INFO_NULL,
447 };
448