40 static void xe_gt_tlb_invalidation_fence_fini(struct xe_gt_tlb_invalidation_fence *fence)  in xe_gt_tlb_invalidation_fence_fini()  argument
42 	if (WARN_ON_ONCE(!fence->gt))  in xe_gt_tlb_invalidation_fence_fini()
45 	xe_pm_runtime_put(gt_to_xe(fence->gt));  in xe_gt_tlb_invalidation_fence_fini()
46 	fence->gt = NULL; /* fini() should be called once */  in xe_gt_tlb_invalidation_fence_fini()
50 __invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence)  in __invalidation_fence_signal()  argument
52 	bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags);  in __invalidation_fence_signal()
54 	trace_xe_gt_tlb_invalidation_fence_signal(xe, fence);  in __invalidation_fence_signal()
55 	xe_gt_tlb_invalidation_fence_fini(fence);  in __invalidation_fence_signal()
56 	dma_fence_signal(&fence->base);  in __invalidation_fence_signal()
58 		dma_fence_put(&fence->base);  in __invalidation_fence_signal()
62 invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence)  in invalidation_fence_signal()  argument
64 	list_del(&fence->link);  in invalidation_fence_signal()
65 	__invalidation_fence_signal(xe, fence);  in invalidation_fence_signal()
68 void xe_gt_tlb_invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence)  in xe_gt_tlb_invalidation_fence_signal()  argument
70 	if (WARN_ON_ONCE(!fence->gt))  in xe_gt_tlb_invalidation_fence_signal()
73 	__invalidation_fence_signal(gt_to_xe(fence->gt), fence);  in xe_gt_tlb_invalidation_fence_signal()
81 	struct xe_gt_tlb_invalidation_fence *fence, *next;  in xe_gt_tlb_fence_timeout()  local
86 	list_for_each_entry_safe(fence, next,  in xe_gt_tlb_fence_timeout()
89 						    fence->invalidation_time);  in xe_gt_tlb_fence_timeout()
94 		trace_xe_gt_tlb_invalidation_fence_timeout(xe, fence);  in xe_gt_tlb_fence_timeout()
95 		xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",  in xe_gt_tlb_fence_timeout()
96 			  fence->seqno, gt->tlb_invalidation.seqno_recv);  in xe_gt_tlb_fence_timeout()
98 		fence->base.error = -ETIME;  in xe_gt_tlb_fence_timeout()
99 		invalidation_fence_signal(xe, fence);  in xe_gt_tlb_fence_timeout()
137 	struct xe_gt_tlb_invalidation_fence *fence, *next;  in xe_gt_tlb_invalidation_reset()  local
150 	 * which are not tracked with an explicit TLB fence, however at this  in xe_gt_tlb_invalidation_reset()
161 	list_for_each_entry_safe(fence, next,  in xe_gt_tlb_invalidation_reset()
163 		invalidation_fence_signal(gt_to_xe(gt), fence);  in xe_gt_tlb_invalidation_reset()
182 				 struct xe_gt_tlb_invalidation_fence *fence,  in send_tlb_invalidation()  argument
190 	xe_gt_assert(gt, fence);  in send_tlb_invalidation()
200 	fence->seqno = seqno;  in send_tlb_invalidation()
201 	trace_xe_gt_tlb_invalidation_fence_send(xe, fence);  in send_tlb_invalidation()
208 		 * We haven't actually published the TLB fence as per  in send_tlb_invalidation()
211 		 * we can just go ahead and signal the fence here.  in send_tlb_invalidation()
214 			__invalidation_fence_signal(xe, fence);  in send_tlb_invalidation()
216 			fence->invalidation_time = ktime_get();  in send_tlb_invalidation()
217 			list_add_tail(&fence->link,  in send_tlb_invalidation()
227 		__invalidation_fence_signal(xe, fence);  in send_tlb_invalidation()
248  * @fence: invalidation fence which will be signal on TLB invalidation
252  * caller can use the invalidation fence to wait for completion.
257 				      struct xe_gt_tlb_invalidation_fence *fence)  in xe_gt_tlb_invalidation_guc()  argument
266 	ret = send_tlb_invalidation(&gt->uc.guc, fence, action,  in xe_gt_tlb_invalidation_guc()
294 		struct xe_gt_tlb_invalidation_fence fence;  in xe_gt_tlb_invalidation_ggtt()  local
297 		xe_gt_tlb_invalidation_fence_init(gt, &fence, true);  in xe_gt_tlb_invalidation_ggtt()
298 		ret = xe_gt_tlb_invalidation_guc(gt, &fence);  in xe_gt_tlb_invalidation_ggtt()
302 		xe_gt_tlb_invalidation_fence_wait(&fence);  in xe_gt_tlb_invalidation_ggtt()
337  * @fence: invalidation fence which will be signal on TLB invalidation
345  * the invalidation fence to wait for completion.
350 				 struct xe_gt_tlb_invalidation_fence *fence,  in xe_gt_tlb_invalidation_range()  argument
359 	xe_gt_assert(gt, fence);  in xe_gt_tlb_invalidation_range()
363 		__invalidation_fence_signal(xe, fence);  in xe_gt_tlb_invalidation_range()
418 	return send_tlb_invalidation(&gt->uc.guc, fence, action, len);  in xe_gt_tlb_invalidation_range()
424  * @fence: invalidation fence which will be signal on TLB invalidation
430  * the invalidation fence to wait for completion.
435 			       struct xe_gt_tlb_invalidation_fence *fence,  in xe_gt_tlb_invalidation_vma()  argument
440 	return xe_gt_tlb_invalidation_range(gt, fence, xe_vma_start(vma),  in xe_gt_tlb_invalidation_vma()
461 	struct xe_gt_tlb_invalidation_fence *fence, *next;  in xe_guc_tlb_invalidation_done_handler()  local
490 	list_for_each_entry_safe(fence, next,  in xe_guc_tlb_invalidation_done_handler()
492 		trace_xe_gt_tlb_invalidation_fence_recv(xe, fence);  in xe_guc_tlb_invalidation_done_handler()
494 		if (!tlb_invalidation_seqno_past(gt, fence->seqno))  in xe_guc_tlb_invalidation_done_handler()
497 		invalidation_fence_signal(xe, fence);  in xe_guc_tlb_invalidation_done_handler()
530  * xe_gt_tlb_invalidation_fence_init - Initialize TLB invalidation fence
532  * @fence: TLB invalidation fence to initialize
533  * @stack: fence is stack variable
535  * Initialize TLB invalidation fence for use. xe_gt_tlb_invalidation_fence_fini
536  * will be automatically called when fence is signalled (all fences must signal),
540 				       struct xe_gt_tlb_invalidation_fence *fence,  in xe_gt_tlb_invalidation_fence_init()  argument
546 	dma_fence_init(&fence->base, &invalidation_fence_ops,  in xe_gt_tlb_invalidation_fence_init()
550 	INIT_LIST_HEAD(&fence->link);  in xe_gt_tlb_invalidation_fence_init()
552 		set_bit(FENCE_STACK_BIT, &fence->base.flags);  in xe_gt_tlb_invalidation_fence_init()
554 		dma_fence_get(&fence->base);  in xe_gt_tlb_invalidation_fence_init()
555 	fence->gt = gt;  in xe_gt_tlb_invalidation_fence_init()