1 /*
2 * Copyright © 2011 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Chris Wilson <[email protected]>
25 *
26 */
27
28 #include <unistd.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <string.h>
32 #include <fcntl.h>
33 #include <inttypes.h>
34 #include <pthread.h>
35 #include <errno.h>
36 #include <sys/stat.h>
37 #include <sys/ioctl.h>
38 #include "drm.h"
39
40 #include "igt.h"
41 #include "igt_sysfs.h"
42 #include "igt_x86.h"
43
44 #ifndef PAGE_SIZE
45 #define PAGE_SIZE 4096
46 #endif
47
48 #define abs(x) ((x) >= 0 ? (x) : -(x))
49
50 static int OBJECT_SIZE = 16*1024*1024;
51
52 static void
set_domain_gtt(int fd,uint32_t handle)53 set_domain_gtt(int fd, uint32_t handle)
54 {
55 gem_set_domain(fd, handle, I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
56 }
57
58 static void *
mmap_bo(int fd,uint32_t handle)59 mmap_bo(int fd, uint32_t handle)
60 {
61 void *ptr;
62
63 ptr = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_READ | PROT_WRITE);
64
65 return ptr;
66 }
67
68 static void *
create_pointer(int fd)69 create_pointer(int fd)
70 {
71 uint32_t handle;
72 void *ptr;
73
74 handle = gem_create(fd, OBJECT_SIZE);
75
76 ptr = mmap_bo(fd, handle);
77
78 gem_close(fd, handle);
79
80 return ptr;
81 }
82
83 static void
test_access(int fd)84 test_access(int fd)
85 {
86 uint32_t handle, flink, handle2;
87 struct drm_i915_gem_mmap_gtt mmap_arg;
88 int fd2;
89
90 handle = gem_create(fd, OBJECT_SIZE);
91 igt_assert(handle);
92
93 fd2 = drm_open_driver(DRIVER_INTEL);
94
95 /* Check that fd1 can mmap. */
96 mmap_arg.handle = handle;
97 do_ioctl(fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg);
98
99 igt_assert(mmap64(0, OBJECT_SIZE, PROT_READ | PROT_WRITE,
100 MAP_SHARED, fd, mmap_arg.offset));
101
102 /* Check that the same offset on the other fd doesn't work. */
103 igt_assert(mmap64(0, OBJECT_SIZE, PROT_READ | PROT_WRITE,
104 MAP_SHARED, fd2, mmap_arg.offset) == MAP_FAILED);
105 igt_assert(errno == EACCES);
106
107 flink = gem_flink(fd, handle);
108 igt_assert(flink);
109 handle2 = gem_open(fd2, flink);
110 igt_assert(handle2);
111
112 /* Recheck that it works after flink. */
113 /* Check that the same offset on the other fd doesn't work. */
114 igt_assert(mmap64(0, OBJECT_SIZE, PROT_READ | PROT_WRITE,
115 MAP_SHARED, fd2, mmap_arg.offset));
116 }
117
118 static void
test_short(int fd)119 test_short(int fd)
120 {
121 struct drm_i915_gem_mmap_gtt mmap_arg;
122 int pages, p;
123
124 mmap_arg.handle = gem_create(fd, OBJECT_SIZE);
125 igt_assert(mmap_arg.handle);
126
127 do_ioctl(fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg);
128 for (pages = 1; pages <= OBJECT_SIZE / PAGE_SIZE; pages <<= 1) {
129 uint8_t *r, *w;
130
131 w = mmap64(0, pages * PAGE_SIZE, PROT_READ | PROT_WRITE,
132 MAP_SHARED, fd, mmap_arg.offset);
133 igt_assert(w != MAP_FAILED);
134
135 r = mmap64(0, pages * PAGE_SIZE, PROT_READ,
136 MAP_SHARED, fd, mmap_arg.offset);
137 igt_assert(r != MAP_FAILED);
138
139 for (p = 0; p < pages; p++) {
140 w[p*PAGE_SIZE] = r[p*PAGE_SIZE];
141 w[p*PAGE_SIZE+(PAGE_SIZE-1)] =
142 r[p*PAGE_SIZE+(PAGE_SIZE-1)];
143 }
144
145 munmap(r, pages * PAGE_SIZE);
146 munmap(w, pages * PAGE_SIZE);
147 }
148 gem_close(fd, mmap_arg.handle);
149 }
150
151 static void
test_copy(int fd)152 test_copy(int fd)
153 {
154 void *src, *dst;
155
156 /* copy from a fresh src to fresh dst to force pagefault on both */
157 src = create_pointer(fd);
158 dst = create_pointer(fd);
159
160 memcpy(dst, src, OBJECT_SIZE);
161 memcpy(src, dst, OBJECT_SIZE);
162
163 munmap(dst, OBJECT_SIZE);
164 munmap(src, OBJECT_SIZE);
165 }
166
167 enum test_read_write {
168 READ_BEFORE_WRITE,
169 READ_AFTER_WRITE,
170 };
171
172 static void
test_read_write(int fd,enum test_read_write order)173 test_read_write(int fd, enum test_read_write order)
174 {
175 uint32_t handle;
176 void *ptr;
177 volatile uint32_t val = 0;
178
179 handle = gem_create(fd, OBJECT_SIZE);
180
181 ptr = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_READ | PROT_WRITE);
182 gem_set_domain(fd, handle, I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
183
184 if (order == READ_BEFORE_WRITE) {
185 val = *(uint32_t *)ptr;
186 *(uint32_t *)ptr = val;
187 } else {
188 *(uint32_t *)ptr = val;
189 val = *(uint32_t *)ptr;
190 }
191
192 gem_close(fd, handle);
193 munmap(ptr, OBJECT_SIZE);
194 }
195
196 static void
test_read_write2(int fd,enum test_read_write order)197 test_read_write2(int fd, enum test_read_write order)
198 {
199 uint32_t handle;
200 void *r, *w;
201 volatile uint32_t val = 0;
202
203 handle = gem_create(fd, OBJECT_SIZE);
204
205 r = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_READ);
206 w = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_READ | PROT_WRITE);
207
208 gem_set_domain(fd, handle, I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
209
210 if (order == READ_BEFORE_WRITE) {
211 val = *(uint32_t *)r;
212 *(uint32_t *)w = val;
213 } else {
214 *(uint32_t *)w = val;
215 val = *(uint32_t *)r;
216 }
217
218 gem_close(fd, handle);
219 munmap(r, OBJECT_SIZE);
220 munmap(w, OBJECT_SIZE);
221 }
222
223 static void
test_write(int fd)224 test_write(int fd)
225 {
226 void *src;
227 uint32_t dst;
228
229 /* copy from a fresh src to fresh dst to force pagefault on both */
230 src = create_pointer(fd);
231 dst = gem_create(fd, OBJECT_SIZE);
232
233 gem_write(fd, dst, 0, src, OBJECT_SIZE);
234
235 gem_close(fd, dst);
236 munmap(src, OBJECT_SIZE);
237 }
238
239 static void
test_wc(int fd)240 test_wc(int fd)
241 {
242 unsigned long gtt_reads, gtt_writes, cpu_writes;
243 uint32_t handle;
244 void *gtt, *cpu;
245
246 handle = gem_create(fd, 4096);
247 cpu = gem_mmap__cpu(fd, handle, 0, 4096, PROT_READ | PROT_WRITE);
248 gem_set_domain(fd, handle, I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
249 gem_close(fd, handle);
250
251 handle = gem_create(fd, 4096);
252 gtt = gem_mmap__gtt(fd, handle, 4096, PROT_READ | PROT_WRITE);
253 gem_set_domain(fd, handle, I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
254 gem_close(fd, handle);
255
256 gtt_reads = 0;
257 igt_for_milliseconds(200) {
258 memcpy(cpu, gtt, 4096);
259 gtt_reads++;
260 }
261 igt_debug("%lu GTT reads in 200us\n", gtt_reads);
262
263 gtt_writes = 0;
264 igt_for_milliseconds(200) {
265 memcpy(gtt, cpu, 4096);
266 gtt_writes++;
267 }
268 igt_debug("%lu GTT writes in 200us\n", gtt_writes);
269
270 if (igt_setup_clflush()) {
271 cpu_writes = 0;
272 igt_for_milliseconds(200) {
273 igt_clflush_range(cpu, 4096);
274 cpu_writes++;
275 }
276 igt_debug("%lu CPU writes in 200us\n", cpu_writes);
277 } else
278 cpu_writes = gtt_writes;
279
280 munmap(cpu, 4096);
281 munmap(gtt, 4096);
282
283 igt_assert_f(gtt_writes > 2*gtt_reads,
284 "Write-Combined writes are expected to be much faster than reads: read=%.2fMiB/s, write=%.2fMiB/s\n",
285 5*gtt_reads/256., 5*gtt_writes/256.);
286
287 igt_assert_f(gtt_writes > cpu_writes/2,
288 "Write-Combined writes are expected to be roughly equivalent to WB writes: WC (gtt)=%.2fMiB/s, WB (cpu)=%.2fMiB/s\n",
289 5*gtt_writes/256., 5*cpu_writes/256.);
290 }
291
mmap_gtt_version(int i915)292 static int mmap_gtt_version(int i915)
293 {
294 int val = 0;
295 struct drm_i915_getparam gp = {
296 gp.param = 40, /* MMAP_GTT_VERSION */
297 gp.value = &val,
298 };
299
300 ioctl(i915, DRM_IOCTL_I915_GETPARAM, &gp);
301 return val;
302 }
303
304 static void
test_pf_nonblock(int i915)305 test_pf_nonblock(int i915)
306 {
307 igt_spin_t *spin;
308 uint32_t *ptr;
309
310 igt_require(mmap_gtt_version(i915) >= 3);
311
312 spin = igt_spin_new(i915);
313
314 igt_set_timeout(1, "initial pagefaulting did not complete within 1s");
315
316 ptr = gem_mmap__gtt(i915, spin->handle, 4096, PROT_WRITE);
317 ptr[256] = 0;
318 munmap(ptr, 4096);
319
320 igt_reset_timeout();
321
322 igt_spin_free(i915, spin);
323 }
324
325 static void
test_isolation(int i915)326 test_isolation(int i915)
327 {
328 struct drm_i915_gem_mmap_gtt mmap_arg;
329 int A = gem_reopen_driver(i915);
330 int B = gem_reopen_driver(i915);
331 uint64_t offset_a, offset_b;
332 uint32_t a, b;
333 void *ptr;
334
335 a = gem_create(A, 4096);
336 b = gem_open(B, gem_flink(A, a));
337
338 mmap_arg.handle = a;
339 do_ioctl(A, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg);
340 offset_a = mmap_arg.offset;
341
342 mmap_arg.handle = b;
343 do_ioctl(B, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg);
344 offset_b = mmap_arg.offset;
345
346 igt_info("A: {fd:%d, handle:%d, offset:%"PRIx64"}\n",
347 A, a, offset_a);
348 igt_info("B: {fd:%d, handle:%d, offset:%"PRIx64"}\n",
349 B, b, offset_b);
350
351 close(B);
352
353 ptr = mmap64(0, 4096, PROT_READ, MAP_SHARED, A, offset_a);
354 igt_assert(ptr != MAP_FAILED);
355 munmap(ptr, 4096);
356
357 close(A);
358
359 ptr = mmap64(0, 4096, PROT_READ, MAP_SHARED, A, offset_a);
360 igt_assert(ptr == MAP_FAILED);
361 }
362
363 static void
test_write_gtt(int fd)364 test_write_gtt(int fd)
365 {
366 uint32_t dst;
367 char *dst_gtt;
368 void *src;
369
370 dst = gem_create(fd, OBJECT_SIZE);
371
372 /* prefault object into gtt */
373 dst_gtt = mmap_bo(fd, dst);
374 set_domain_gtt(fd, dst);
375 memset(dst_gtt, 0, OBJECT_SIZE);
376 munmap(dst_gtt, OBJECT_SIZE);
377
378 src = create_pointer(fd);
379
380 gem_write(fd, dst, 0, src, OBJECT_SIZE);
381
382 gem_close(fd, dst);
383 munmap(src, OBJECT_SIZE);
384 }
385
is_coherent(int i915)386 static bool is_coherent(int i915)
387 {
388 int val = 1; /* by default, we assume GTT is coherent, hence the test */
389 struct drm_i915_getparam gp = {
390 gp.param = 52, /* GTT_COHERENT */
391 gp.value = &val,
392 };
393
394 ioctl(i915, DRM_IOCTL_I915_GETPARAM, &gp);
395 return val;
396 }
397
398 static void
test_coherency(int fd)399 test_coherency(int fd)
400 {
401 uint32_t handle;
402 uint32_t *gtt, *cpu;
403 int i;
404
405 igt_require(is_coherent(fd));
406 igt_require(igt_setup_clflush());
407
408 handle = gem_create(fd, OBJECT_SIZE);
409
410 gtt = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_READ | PROT_WRITE);
411 cpu = gem_mmap__cpu(fd, handle, 0, OBJECT_SIZE, PROT_READ | PROT_WRITE);
412 set_domain_gtt(fd, handle);
413
414 /* On byt/bsw/bxt this detects an interesting behaviour where the
415 * CPU cannot flush the iobar and so the read may bypass the write.
416 * https://bugs.freedesktop.org/show_bug.cgi?id=94314
417 */
418 for (i = 0; i < OBJECT_SIZE / 64; i++) {
419 int x = 16*i + (i%16);
420 gtt[x] = i;
421 igt_clflush_range(&cpu[x], sizeof(cpu[x]));
422 igt_assert_eq(cpu[x], i);
423 }
424
425 munmap(cpu, OBJECT_SIZE);
426 munmap(gtt, OBJECT_SIZE);
427 gem_close(fd, handle);
428 }
429
430 static void
test_clflush(int fd)431 test_clflush(int fd)
432 {
433 uint32_t handle;
434 uint32_t *gtt;
435
436 igt_require(igt_setup_clflush());
437
438 handle = gem_create(fd, OBJECT_SIZE);
439
440 gtt = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_READ | PROT_WRITE);
441 set_domain_gtt(fd, handle);
442
443 igt_clflush_range(gtt, OBJECT_SIZE);
444
445 munmap(gtt, OBJECT_SIZE);
446 gem_close(fd, handle);
447 }
448
449 static void
test_hang(int fd)450 test_hang(int fd)
451 {
452 const uint32_t patterns[] = {
453 0, 0xaaaaaaaa, 0x55555555, 0xcccccccc,
454 };
455 const int ncpus = sysconf(_SC_NPROCESSORS_ONLN);
456 struct {
457 bool done;
458 bool error;
459 } *control;
460 unsigned long count;
461 igt_hang_t hang;
462 int dir;
463
464 hang = igt_allow_hang(fd, 0, 0);
465 igt_require(igt_sysfs_set_parameter(fd, "reset", "1")); /* global */
466
467 control = mmap(NULL, 4096, PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0);
468 igt_assert(control != MAP_FAILED);
469
470 igt_fork(child, ncpus) {
471 int last_pattern = 0;
472 int next_pattern = 1;
473 uint32_t *gtt[2];
474
475 for (int i = 0; i < ARRAY_SIZE(gtt); i++) {
476 uint32_t handle;
477
478 handle = gem_create(fd, OBJECT_SIZE);
479 gem_set_tiling(fd, handle, I915_TILING_X + i, 2048);
480
481 gtt[i] = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_WRITE);
482 set_domain_gtt(fd, handle);
483 gem_close(fd, handle);
484 }
485
486 while (!READ_ONCE(control->done)) {
487 for (int i = 0; i < OBJECT_SIZE / 64; i++) {
488 const unsigned int x = 16 * i + ( i% 16);
489 uint32_t expected = patterns[last_pattern];
490 uint32_t found[2];
491
492 found[0] = READ_ONCE(gtt[0][x]);
493 found[1] = READ_ONCE(gtt[1][x]);
494
495 if (found[0] != expected ||
496 found[1] != expected) {
497 igt_warn("child[%d] found (%x, %x), expecting %x\n",
498 child,
499 found[0], found[1],
500 expected);
501 control->error = true;
502 exit(0);
503 }
504
505 gtt[0][x] = patterns[next_pattern];
506 gtt[1][x] = patterns[next_pattern];
507 }
508
509 last_pattern = next_pattern;
510 next_pattern = (next_pattern + 1) % ARRAY_SIZE(patterns);
511 }
512 }
513
514 count = 0;
515 dir = igt_debugfs_dir(fd);
516 igt_until_timeout(5) {
517 igt_sysfs_set(dir, "i915_wedged", "-1");
518 if (READ_ONCE(control->error))
519 break;
520 count++;
521 }
522 close(dir);
523 igt_info("%lu resets\n", count);
524
525 control->done = true;
526 igt_waitchildren_timeout(2, NULL);
527
528 igt_assert(!control->error);
529 munmap(control, 4096);
530
531 igt_disallow_hang(fd, hang);
532 }
533
min_tile_width(uint32_t devid,int tiling)534 static int min_tile_width(uint32_t devid, int tiling)
535 {
536 if (tiling < 0) {
537 if (intel_gen(devid) >= 4)
538 return 4096 - min_tile_width(devid, -tiling);
539 else
540 return 1024;
541
542 }
543
544 if (intel_gen(devid) == 2)
545 return 128;
546 else if (tiling == I915_TILING_X)
547 return 512;
548 else if (IS_915(devid))
549 return 512;
550 else
551 return 128;
552 }
553
max_tile_width(uint32_t devid,int tiling)554 static int max_tile_width(uint32_t devid, int tiling)
555 {
556 if (tiling < 0) {
557 if (intel_gen(devid) >= 4)
558 return 4096 + min_tile_width(devid, -tiling);
559 else
560 return 2048;
561 }
562
563 if (intel_gen(devid) >= 7)
564 return 256 << 10;
565 else if (intel_gen(devid) >= 4)
566 return 128 << 10;
567 else
568 return 8 << 10;
569 }
570
known_swizzling(int fd,uint32_t handle)571 static bool known_swizzling(int fd, uint32_t handle)
572 {
573 struct drm_i915_gem_get_tiling2 {
574 uint32_t handle;
575 uint32_t tiling_mode;
576 uint32_t swizzle_mode;
577 uint32_t phys_swizzle_mode;
578 } arg = {
579 .handle = handle,
580 };
581 #define DRM_IOCTL_I915_GEM_GET_TILING2 DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_GET_TILING, struct drm_i915_gem_get_tiling2)
582
583 if (igt_ioctl(fd, DRM_IOCTL_I915_GEM_GET_TILING2, &arg))
584 return false;
585
586 return arg.phys_swizzle_mode == arg.swizzle_mode;
587 }
588
589 static void
test_huge_bo(int fd,int huge,int tiling)590 test_huge_bo(int fd, int huge, int tiling)
591 {
592 uint32_t bo;
593 char *ptr;
594 char *tiled_pattern;
595 char *linear_pattern;
596 uint64_t size, last_offset;
597 uint32_t devid = intel_get_drm_devid(fd);
598 int pitch = min_tile_width(devid, tiling);
599 int i;
600
601 switch (huge) {
602 case -1:
603 size = gem_mappable_aperture_size() / 2;
604
605 /* Power of two fence size, natural fence
606 * alignment, and the guard page at the end
607 * gtt means that if the entire gtt is
608 * mappable, we can't usually fit in a tiled
609 * object half the size of the gtt. Let's use
610 * a quarter size one instead.
611 */
612 if (tiling &&
613 intel_gen(intel_get_drm_devid(fd)) < 4 &&
614 size >= gem_global_aperture_size(fd) / 2)
615 size /= 2;
616 break;
617 case 0:
618 size = gem_mappable_aperture_size() + PAGE_SIZE;
619 break;
620 default:
621 size = gem_global_aperture_size(fd) + PAGE_SIZE;
622 break;
623 }
624 intel_require_memory(1, size, CHECK_RAM);
625
626 last_offset = size - PAGE_SIZE;
627
628 /* Create pattern */
629 bo = gem_create(fd, PAGE_SIZE);
630 if (tiling)
631 igt_require(__gem_set_tiling(fd, bo, tiling, pitch) == 0);
632 igt_require(known_swizzling(fd, bo));
633
634 linear_pattern = gem_mmap__gtt(fd, bo, PAGE_SIZE,
635 PROT_READ | PROT_WRITE);
636 for (i = 0; i < PAGE_SIZE; i++)
637 linear_pattern[i] = i;
638 tiled_pattern = gem_mmap__cpu(fd, bo, 0, PAGE_SIZE, PROT_READ);
639
640 gem_set_domain(fd, bo, I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT, 0);
641 gem_close(fd, bo);
642
643 bo = gem_create(fd, size);
644 if (tiling)
645 igt_require(__gem_set_tiling(fd, bo, tiling, pitch) == 0);
646
647 /* Initialise first/last page through CPU mmap */
648 ptr = gem_mmap__cpu(fd, bo, 0, size, PROT_READ | PROT_WRITE);
649 memcpy(ptr, tiled_pattern, PAGE_SIZE);
650 memcpy(ptr + last_offset, tiled_pattern, PAGE_SIZE);
651 munmap(ptr, size);
652
653 /* Obtain mapping for the object through GTT. */
654 ptr = __gem_mmap__gtt(fd, bo, size, PROT_READ | PROT_WRITE);
655 igt_require_f(ptr, "Huge BO GTT mapping not supported.\n");
656
657 set_domain_gtt(fd, bo);
658
659 /* Access through GTT should still provide the CPU written values. */
660 igt_assert(memcmp(ptr , linear_pattern, PAGE_SIZE) == 0);
661 igt_assert(memcmp(ptr + last_offset, linear_pattern, PAGE_SIZE) == 0);
662
663 gem_set_tiling(fd, bo, I915_TILING_NONE, 0);
664
665 igt_assert(memcmp(ptr , tiled_pattern, PAGE_SIZE) == 0);
666 igt_assert(memcmp(ptr + last_offset, tiled_pattern, PAGE_SIZE) == 0);
667
668 munmap(ptr, size);
669
670 gem_close(fd, bo);
671 munmap(tiled_pattern, PAGE_SIZE);
672 munmap(linear_pattern, PAGE_SIZE);
673 }
674
copy_wc_page(void * dst,const void * src)675 static void copy_wc_page(void *dst, const void *src)
676 {
677 igt_memcpy_from_wc(dst, src, PAGE_SIZE);
678 }
679
tile_row_size(int tiling,unsigned int stride)680 static unsigned int tile_row_size(int tiling, unsigned int stride)
681 {
682 if (tiling < 0)
683 tiling = -tiling;
684
685 return stride * (tiling == I915_TILING_Y ? 32 : 8);
686 }
687
688 #define rounddown(x, y) (x - (x%y))
689
690 static void
test_huge_copy(int fd,int huge,int tiling_a,int tiling_b,int ncpus)691 test_huge_copy(int fd, int huge, int tiling_a, int tiling_b, int ncpus)
692 {
693 const uint32_t devid = intel_get_drm_devid(fd);
694 uint64_t huge_object_size, i;
695 unsigned mode = CHECK_RAM;
696
697 igt_fail_on_f(intel_gen(devid) >= 11 && ncpus > 1,
698 "Please adjust your expectations, https://bugs.freedesktop.org/show_bug.cgi?id=110882\n");
699
700 switch (huge) {
701 case -2:
702 huge_object_size = gem_mappable_aperture_size() / 4;
703 break;
704 case -1:
705 huge_object_size = gem_mappable_aperture_size() / 2;
706 break;
707 case 0:
708 huge_object_size = gem_mappable_aperture_size() + PAGE_SIZE;
709 break;
710 case 1:
711 huge_object_size = gem_global_aperture_size(fd) + PAGE_SIZE;
712 break;
713 default:
714 huge_object_size = (intel_get_total_ram_mb() << 19) + PAGE_SIZE;
715 mode |= CHECK_SWAP;
716 break;
717 }
718 intel_require_memory(2*ncpus, huge_object_size, mode);
719
720 igt_fork(child, ncpus) {
721 uint64_t valid_size = huge_object_size;
722 uint32_t bo[2];
723 char *a, *b;
724
725 bo[0] = gem_create(fd, huge_object_size);
726 if (tiling_a) {
727 igt_require(__gem_set_tiling(fd, bo[0], abs(tiling_a), min_tile_width(devid, tiling_a)) == 0);
728 valid_size = rounddown(valid_size, tile_row_size(tiling_a, min_tile_width(devid, tiling_a)));
729 }
730 a = __gem_mmap__gtt(fd, bo[0], huge_object_size, PROT_READ | PROT_WRITE);
731 igt_require(a);
732
733 bo[1] = gem_create(fd, huge_object_size);
734 if (tiling_b) {
735 igt_require(__gem_set_tiling(fd, bo[1], abs(tiling_b), max_tile_width(devid, tiling_b)) == 0);
736 valid_size = rounddown(valid_size, tile_row_size(tiling_b, max_tile_width(devid, tiling_b)));
737 }
738 b = __gem_mmap__gtt(fd, bo[1], huge_object_size, PROT_READ | PROT_WRITE);
739 igt_require(b);
740
741 gem_set_domain(fd, bo[0], I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
742 for (i = 0; i < valid_size / PAGE_SIZE; i++) {
743 uint32_t *ptr = (uint32_t *)(a + PAGE_SIZE*i);
744 for (int j = 0; j < PAGE_SIZE/4; j++)
745 ptr[j] = i + j;
746 igt_progress("Writing a ", i, valid_size / PAGE_SIZE);
747 }
748
749 gem_set_domain(fd, bo[1], I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
750 for (i = 0; i < valid_size / PAGE_SIZE; i++) {
751 uint32_t *ptr = (uint32_t *)(b + PAGE_SIZE*i);
752 for (int j = 0; j < PAGE_SIZE/4; j++)
753 ptr[j] = ~(i + j);
754 igt_progress("Writing b ", i, valid_size / PAGE_SIZE);
755 }
756
757 for (i = 0; i < valid_size / PAGE_SIZE; i++) {
758 uint32_t *A = (uint32_t *)(a + PAGE_SIZE*i);
759 uint32_t *B = (uint32_t *)(b + PAGE_SIZE*i);
760 uint32_t A_tmp[PAGE_SIZE/sizeof(uint32_t)];
761 uint32_t B_tmp[PAGE_SIZE/sizeof(uint32_t)];
762
763 copy_wc_page(A_tmp, A);
764 copy_wc_page(B_tmp, B);
765 for (int j = 0; j < PAGE_SIZE/4; j++)
766 if ((i + j) & 1)
767 A_tmp[j] = B_tmp[j];
768 else
769 B_tmp[j] = A_tmp[j];
770
771 gem_set_domain(fd, bo[0], I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
772 memcpy(A, A_tmp, PAGE_SIZE);
773
774 gem_set_domain(fd, bo[1], I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
775 memcpy(B, B_tmp, PAGE_SIZE);
776
777 igt_progress("Copying a<->b ", i, valid_size / PAGE_SIZE);
778 }
779
780 gem_close(fd, bo[0]);
781 gem_close(fd, bo[1]);
782
783 for (i = 0; i < valid_size / PAGE_SIZE; i++) {
784 uint32_t page[PAGE_SIZE/sizeof(uint32_t)];
785 copy_wc_page(page, a + PAGE_SIZE*i);
786 for (int j = 0; j < PAGE_SIZE/sizeof(uint32_t); j++)
787 if ((i + j) & 1)
788 igt_assert_eq_u32(page[j], ~(i + j));
789 else
790 igt_assert_eq_u32(page[j], i + j);
791 igt_progress("Checking a ", i, valid_size / PAGE_SIZE);
792 }
793 munmap(a, huge_object_size);
794
795 for (i = 0; i < valid_size / PAGE_SIZE; i++) {
796 uint32_t page[PAGE_SIZE/sizeof(uint32_t)];
797 copy_wc_page(page, b + PAGE_SIZE*i);
798 for (int j = 0; j < PAGE_SIZE/sizeof(uint32_t); j++)
799 if ((i + j) & 1)
800 igt_assert_eq_u32(page[j], ~(i + j));
801 else
802 igt_assert_eq_u32(page[j], i + j);
803 igt_progress("Checking b ", i, valid_size / PAGE_SIZE);
804 }
805 munmap(b, huge_object_size);
806 }
807 igt_waitchildren();
808 }
809
810 static void
test_read(int fd)811 test_read(int fd)
812 {
813 void *dst;
814 uint32_t src;
815
816 /* copy from a fresh src to fresh dst to force pagefault on both */
817 dst = create_pointer(fd);
818 src = gem_create(fd, OBJECT_SIZE);
819
820 gem_read(fd, src, 0, dst, OBJECT_SIZE);
821
822 gem_close(fd, src);
823 munmap(dst, OBJECT_SIZE);
824 }
825
826 static void
test_write_cpu_read_gtt(int fd)827 test_write_cpu_read_gtt(int fd)
828 {
829 uint32_t handle;
830 uint32_t *src, *dst;
831
832 igt_require(gem_has_llc(fd));
833
834 handle = gem_create(fd, OBJECT_SIZE);
835
836 dst = gem_mmap__gtt(fd, handle, OBJECT_SIZE, PROT_READ);
837
838 src = gem_mmap__cpu(fd, handle, 0, OBJECT_SIZE, PROT_WRITE);
839
840 gem_close(fd, handle);
841
842 memset(src, 0xaa, OBJECT_SIZE);
843 igt_assert(memcmp(dst, src, OBJECT_SIZE) == 0);
844
845 munmap(src, OBJECT_SIZE);
846 munmap(dst, OBJECT_SIZE);
847 }
848
849 struct thread_fault_concurrent {
850 pthread_t thread;
851 int id;
852 uint32_t **ptr;
853 };
854
855 static void *
thread_fault_concurrent(void * closure)856 thread_fault_concurrent(void *closure)
857 {
858 struct thread_fault_concurrent *t = closure;
859 uint32_t val = 0;
860 int n;
861
862 for (n = 0; n < 32; n++) {
863 if (n & 1)
864 *t->ptr[(n + t->id) % 32] = val;
865 else
866 val = *t->ptr[(n + t->id) % 32];
867 }
868
869 return NULL;
870 }
871
872 static void
test_fault_concurrent(int fd)873 test_fault_concurrent(int fd)
874 {
875 uint32_t *ptr[32];
876 struct thread_fault_concurrent thread[64];
877 int n;
878
879 for (n = 0; n < 32; n++) {
880 ptr[n] = create_pointer(fd);
881 }
882
883 for (n = 0; n < 64; n++) {
884 thread[n].ptr = ptr;
885 thread[n].id = n;
886 pthread_create(&thread[n].thread, NULL, thread_fault_concurrent, &thread[n]);
887 }
888
889 for (n = 0; n < 64; n++)
890 pthread_join(thread[n].thread, NULL);
891
892 for (n = 0; n < 32; n++) {
893 munmap(ptr[n], OBJECT_SIZE);
894 }
895 }
896
897 static void
run_without_prefault(int fd,void (* func)(int fd))898 run_without_prefault(int fd,
899 void (*func)(int fd))
900 {
901 igt_disable_prefault();
902 func(fd);
903 igt_enable_prefault();
904 }
905
mmap_ioctl(int i915,struct drm_i915_gem_mmap_gtt * arg)906 static int mmap_ioctl(int i915, struct drm_i915_gem_mmap_gtt *arg)
907 {
908 int err = 0;
909
910 if (igt_ioctl(i915, DRM_IOCTL_I915_GEM_MMAP_GTT, arg))
911 err = -errno;
912
913 errno = 0;
914 return err;
915 }
916
917 int fd;
918
919 igt_main
920 {
921 if (igt_run_in_simulation())
922 OBJECT_SIZE = 1 * 1024 * 1024;
923
924 igt_fixture
925 fd = drm_open_driver(DRIVER_INTEL);
926
927 igt_subtest("bad-object") {
928 uint32_t real_handle = gem_create(fd, 4096);
929 uint32_t handles[20];
930 size_t i = 0, len;
931
932 handles[i++] = 0xdeadbeef;
933 for(int bit = 0; bit < 16; bit++)
934 handles[i++] = real_handle | (1 << (bit + 16));
935 handles[i++] = real_handle + 1;
936 len = i;
937
938 for (i = 0; i < len; ++i) {
939 struct drm_i915_gem_mmap_gtt arg = {
940 .handle = handles[i],
941 };
942 igt_assert_eq(mmap_ioctl(fd, &arg), -ENOENT);
943 }
944
945 gem_close(fd, real_handle);
946 }
947
948 igt_subtest("basic")
949 test_access(fd);
950 igt_subtest("basic-short")
951 test_short(fd);
952 igt_subtest("basic-copy")
953 test_copy(fd);
954 igt_subtest("basic-read")
955 test_read(fd);
956 igt_subtest("basic-write")
957 test_write(fd);
958 igt_subtest("basic-write-gtt")
959 test_write_gtt(fd);
960 igt_subtest("coherency")
961 test_coherency(fd);
962 igt_subtest("clflush")
963 test_clflush(fd);
964 igt_subtest("hang")
965 test_hang(fd);
966 igt_subtest("basic-read-write")
967 test_read_write(fd, READ_BEFORE_WRITE);
968 igt_subtest("basic-write-read")
969 test_read_write(fd, READ_AFTER_WRITE);
970 igt_subtest("basic-read-write-distinct")
971 test_read_write2(fd, READ_BEFORE_WRITE);
972 igt_subtest("basic-write-read-distinct")
973 test_read_write2(fd, READ_AFTER_WRITE);
974 igt_subtest("fault-concurrent")
975 test_fault_concurrent(fd);
976 igt_subtest("basic-read-no-prefault")
977 run_without_prefault(fd, test_read);
978 igt_subtest("basic-write-no-prefault")
979 run_without_prefault(fd, test_write);
980 igt_subtest("basic-write-gtt-no-prefault")
981 run_without_prefault(fd, test_write_gtt);
982 igt_subtest("basic-write-cpu-read-gtt")
983 test_write_cpu_read_gtt(fd);
984 igt_subtest("basic-wc")
985 test_wc(fd);
986 igt_subtest("isolation")
987 test_isolation(fd);
988 igt_subtest("pf-nonblock")
989 test_pf_nonblock(fd);
990
991 igt_subtest("basic-small-bo")
992 test_huge_bo(fd, -1, I915_TILING_NONE);
993 igt_subtest("basic-small-bo-tiledX")
994 test_huge_bo(fd, -1, I915_TILING_X);
995 igt_subtest("basic-small-bo-tiledY")
996 test_huge_bo(fd, -1, I915_TILING_Y);
997
998 igt_subtest("big-bo")
999 test_huge_bo(fd, 0, I915_TILING_NONE);
1000 igt_subtest("big-bo-tiledX")
1001 test_huge_bo(fd, 0, I915_TILING_X);
1002 igt_subtest("big-bo-tiledY")
1003 test_huge_bo(fd, 0, I915_TILING_Y);
1004
1005 igt_subtest("huge-bo")
1006 test_huge_bo(fd, 1, I915_TILING_NONE);
1007 igt_subtest("huge-bo-tiledX")
1008 test_huge_bo(fd, 1, I915_TILING_X);
1009 igt_subtest("huge-bo-tiledY")
1010 test_huge_bo(fd, 1, I915_TILING_Y);
1011
1012 igt_subtest_group {
1013 const struct copy_size {
1014 const char *prefix;
1015 int size;
1016 } copy_sizes[] = {
1017 { "basic-small", -2 },
1018 { "medium", -1 },
1019 { "big", 0 },
1020 { "huge", 1 },
1021 { "swap", 2 },
1022 { }
1023 };
1024 const struct copy_mode {
1025 const char *suffix;
1026 int tiling_x, tiling_y;
1027 } copy_modes[] = {
1028 { "", I915_TILING_NONE, I915_TILING_NONE},
1029 { "-XY", I915_TILING_X, I915_TILING_Y},
1030 { "-odd", -I915_TILING_X, -I915_TILING_Y},
1031 {}
1032 };
1033 const int ncpus = sysconf(_SC_NPROCESSORS_ONLN);
1034
1035 for (const struct copy_size *s = copy_sizes; s->prefix; s++)
1036 for (const struct copy_mode *m = copy_modes; m->suffix; m++) {
1037 igt_subtest_f("%s-copy%s", s->prefix, m->suffix)
1038 test_huge_copy(fd,
1039 s->size,
1040 m->tiling_x,
1041 m->tiling_y,
1042 1);
1043
1044 igt_subtest_f("forked-%s-copy%s", s->prefix, m->suffix)
1045 test_huge_copy(fd,
1046 s->size,
1047 m->tiling_x,
1048 m->tiling_y,
1049 ncpus);
1050 }
1051 }
1052
1053
1054 igt_fixture
1055 close(fd);
1056 }
1057