1 // SPDX-License-Identifier: MIT
25 	drm_##_level(&memirq_to_xe(m)->drm, "MEMIRQ%u: " _fmt,	\
26 		     memirq_to_tile(m)->id, ##__VA_ARGS__)
54  * DOC: Memory Based Interrupts
56  * MMIO register based interrupts infrastructure used for non-virtualized mode
57  * or SRIOV-8 (which supports 8 Virtual Functions) does not scale efficiently
59  * containers. Memory based interrupt status reporting provides an efficient
62  * For memory based interrupt status reporting hardware sequence is:
65  *    be mapped to system memory and must be marked as un-cacheable (UC) on
71  * DOC: Memory Based Interrupts Page Layout
73  * `Memory Based Interrupts`_ requires three different objects, which are
74  * called "page" in the specs, even if they aren't page-sized or aligned.
76  * To simplify the code we allocate a single page size object and then use
80  * - _`Interrupt Status Report Page`: this page contains the interrupt
88  * - _`Interrupt Source Report Page`: this is the equivalent of the
93  * - Interrupt Mask: the HW needs a location to fetch the interrupt
95  *   the next available space in the interrupt page.
99  *   0x0000   +===========+  <== Interrupt Status Report Page
101  *            |           |     ____ +----+----------------+
103  *            +-----------+ __/      |  1 |                |
105  *            +-----------+   \      |    | WAIT SEMAPHORE |
107  *            |           |          +----+----------------+
109  *   0x0400   +===========+  <== Interrupt Source Report Page
117  *            +-----------+
120  * MSI-X use case
122  * When using MSI-X, hw engines report interrupt status and source to engine
127  * - Interrupt status should be 4KiB aligned
128  * - Interrupt source should be 64 bytes aligned
130  * To accommodate this, we duplicate the memirq page layout above -
131  * allocating a page for each engine instance and pass this page in the LRC.
132  * Note that the same page can be reused for different engine types.
133  * For example, an LRC executing on CCS #x will have pointers to page #x,
138  *   0x0000   +==============================+  <== page for instance 0 (BCS0, CCS0, etc.)
139  *            | Interrupt Status Report Page |
141  *            | Interrupt Source Report Page |
146  *   0x1000   +==============================+  <== page for instance 1 (BCS1, CCS1, etc.)
147  *            | Interrupt Status Report Page |
149  *            | Interrupt Source Report Page |
152  *   0x2000   +==============================+  <== page for instance 2 (BCS2, CCS2, etc.)
161 	 * When the HW engines are configured to use MSI-X,  in hw_reports_to_instance_zero()
192 	memirq_assert(memirq, !memirq->bo);  in memirq_alloc_pages()
194 	iosys_map_memset(&bo->vmap, 0, 0, bo_size);  in memirq_alloc_pages()
196 	memirq->bo = bo;  in memirq_alloc_pages()
197 	memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET(0));  in memirq_alloc_pages()
198 	memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET(0));  in memirq_alloc_pages()
199 	memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET);  in memirq_alloc_pages()
201 	memirq_assert(memirq, !memirq->source.is_iomem);  in memirq_alloc_pages()
202 	memirq_assert(memirq, !memirq->status.is_iomem);  in memirq_alloc_pages()
203 	memirq_assert(memirq, !memirq->mask.is_iomem);  in memirq_alloc_pages()
205 	memirq_debug(memirq, "page offsets: bo %#x bo_size %zu source %#x status %#x\n",  in memirq_alloc_pages()
212 	memirq_err(memirq, "Failed to allocate memirq page (%pe)\n", ERR_PTR(err));  in memirq_alloc_pages()
218 	iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0);  in memirq_set_enable()
220 	memirq->enabled = enable;  in memirq_set_enable()
224  * xe_memirq_init - Initialize data used by `Memory Based Interrupts`_.
227  * Allocate `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_
228  * used by `Memory Based Interrupts`_.
233  * If `Memory Based Interrupts`_ are not used this function will return 0.
258 	memirq_assert(memirq, memirq->bo);  in __memirq_source_page()
261 	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(instance);  in __memirq_source_page()
265  * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_.
267  * @hwe: the hw engine for which we want the report page
269  * Shall be called when `Memory Based Interrupts`_ are used
272  * Return: GGTT's offset of the `Interrupt Source Report Page`_.
278 	return __memirq_source_page(memirq, hwe->instance);  in xe_memirq_source_ptr()
284 	memirq_assert(memirq, memirq->bo);  in __memirq_status_page()
287 	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(instance);  in __memirq_status_page()
291  * xe_memirq_status_ptr - Get GGTT's offset of the `Interrupt Status Report Page`_.
293  * @hwe: the hw engine for which we want the report page
295  * Shall be called when `Memory Based Interrupts`_ are used
298  * Return: GGTT's offset of the `Interrupt Status Report Page`_.
304 	return __memirq_status_page(memirq, hwe->instance);  in xe_memirq_status_ptr()
308  * xe_memirq_enable_ptr - Get GGTT's offset of the Interrupt Enable Mask.
311  * Shall be called when `Memory Based Interrupts`_ are used
319 	memirq_assert(memirq, memirq->bo);  in xe_memirq_enable_ptr()
321 	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_ENABLE_OFFSET;  in xe_memirq_enable_ptr()
325  * xe_memirq_init_guc - Prepare GuC for `Memory Based Interrupts`_.
329  * Register `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_
330  * to be used by the GuC when `Memory Based Interrupts`_ are required.
332  * Shall be called when `Memory Based Interrupts`_ are used
368  * xe_memirq_reset - Disable processing of `Memory Based Interrupts`_.
374  * `Memory Based Interrupts`_.
380 	if (memirq->bo)  in xe_memirq_reset()
385  * xe_memirq_postinstall - Enable processing of `Memory Based Interrupts`_.
391  * `Memory Based Interrupts`_.
397 	if (memirq->bo)  in xe_memirq_postinstall()
421 	memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr);  in memirq_dispatch_engine()
423 	if (memirq_received(memirq, status, ilog2(GT_RENDER_USER_INTERRUPT), hwe->name))  in memirq_dispatch_engine()
432 	memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr);  in memirq_dispatch_guc()
442  * xe_memirq_hwe_handler - Check and process interrupts for a specific HW engine.
446  * This function reads and dispatches `Memory Based Interrupts` for the provided HW engine.
450 	u16 offset = hwe->irq_offset;  in xe_memirq_hwe_handler()
451 	u16 instance = hw_reports_to_instance_zero(memirq) ? hwe->instance : 0;  in xe_memirq_hwe_handler()
452 	struct iosys_map src_offset = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,  in xe_memirq_hwe_handler()
457 			IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,  in xe_memirq_hwe_handler()
464  * xe_memirq_handler - The `Memory Based Interrupts`_ Handler.
467  * This function reads and dispatches `Memory Based Interrupts`.
479 	if (!memirq->bo)  in xe_memirq_handler()
482 	memirq_assert(memirq, !memirq->source.is_iomem);  in xe_memirq_handler()
483 	memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr);  in xe_memirq_handler()
484 	memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32);  in xe_memirq_handler()
487 		if (gt->tile != tile)  in xe_memirq_handler()
496 	if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) {  in xe_memirq_handler()
497 		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16);  in xe_memirq_handler()
498 		memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc);  in xe_memirq_handler()
501 	if (!tile->media_gt)  in xe_memirq_handler()
504 	if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) {  in xe_memirq_handler()
505 		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16);  in xe_memirq_handler()
506 		memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc);  in xe_memirq_handler()