50  * for GPU/CPU synchronization.  When the fence is written,
51  * it is expected that all buffers associated with that fence
59  * radeon_fence_write - write a fence value
63  * @ring: ring index the fence is associated with
65  * Writes a fence value to memory or a scratch register (all asics).
80  * radeon_fence_read - read a fence value
83  * @ring: ring index the fence is associated with
85  * Reads a fence value from memory or a scratch register (all asics).
86  * Returns the value of the fence read from memory or register.
124  * radeon_fence_emit - emit a fence on the requested ring
127  * @fence: radeon fence object
128  * @ring: ring index the fence is associated with
130  * Emits a fence command on the requested ring (all asics).
134 		      struct radeon_fence **fence,  in radeon_fence_emit()  argument
140 	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);  in radeon_fence_emit()
141 	if ((*fence) == NULL)  in radeon_fence_emit()
144 	(*fence)->rdev = rdev;  in radeon_fence_emit()
145 	(*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring];  in radeon_fence_emit()
146 	(*fence)->ring = ring;  in radeon_fence_emit()
147 	(*fence)->is_vm_update = false;  in radeon_fence_emit()
148 	dma_fence_init(&(*fence)->base, &radeon_fence_ops,  in radeon_fence_emit()
152 	radeon_fence_ring_emit(rdev, ring, *fence);  in radeon_fence_emit()
153 	trace_radeon_fence_emit(rdev_to_drm(rdev), ring, (*fence)->seq);  in radeon_fence_emit()
162  * for the fence locking itself, so unlocked variants are used for
168 	struct radeon_fence *fence;  in radeon_fence_check_signaled()  local
171 	fence = container_of(wait, struct radeon_fence, fence_wake);  in radeon_fence_check_signaled()
177 	seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq);  in radeon_fence_check_signaled()
178 	if (seq >= fence->seq) {  in radeon_fence_check_signaled()
179 		dma_fence_signal_locked(&fence->base);  in radeon_fence_check_signaled()
180 		radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring);  in radeon_fence_check_signaled()
181 		__remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake);  in radeon_fence_check_signaled()
182 		dma_fence_put(&fence->base);  in radeon_fence_check_signaled()
188  * radeon_fence_activity - check for fence activity
191  * @ring: ring index the fence is associated with
193  * Checks the current fence value and calculates the last
194  * signalled fence value. Returns true if activity occured
210 	 * continuously new fence signaled ie radeon_fence_read needs  in radeon_fence_activity()
238 		 * checking if a fence is signaled as it means that the  in radeon_fence_activity()
245 			 * fact that we might have set an older fence  in radeon_fence_activity()
264  * Checks for fence activity and if there is none probe
299 …dev_warn(rdev->dev, "GPU lockup (current fence id 0x%016llx last fence id 0x%016llx on ring %d)\n",  in radeon_fence_check_lockup()
311  * radeon_fence_process - process a fence
314  * @ring: ring index the fence is associated with
316  * Checks the current fence value and wakes the fence queue
326  * radeon_fence_seq_signaled - check if a fence sequence number has signaled
330  * @ring: ring index the fence is associated with
332  * Check if the last signaled fence sequnce number is >= the requested
334  * Returns true if the fence has signaled (current fence value
335  * is >= requested value) or false if it has not (current fence
355 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_is_signaled()  local
356 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_is_signaled()
357 	unsigned int ring = fence->ring;  in radeon_fence_is_signaled()
358 	u64 seq = fence->seq;  in radeon_fence_is_signaled()
374  * radeon_fence_enable_signaling - enable signalling on fence
375  * @f: fence
378  * to fence_queue that checks if this fence is signaled, and if so it
379  * signals the fence and removes itself.
383 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_enable_signaling()  local
384 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_enable_signaling()
386 	if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq)  in radeon_fence_enable_signaling()
390 		radeon_irq_kms_sw_irq_get(rdev, fence->ring);  in radeon_fence_enable_signaling()
392 		if (radeon_fence_activity(rdev, fence->ring))  in radeon_fence_enable_signaling()
395 		/* did fence get signaled after we enabled the sw irq? */  in radeon_fence_enable_signaling()
396 		if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) {  in radeon_fence_enable_signaling()
397 			radeon_irq_kms_sw_irq_put(rdev, fence->ring);  in radeon_fence_enable_signaling()
405 		if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring))  in radeon_fence_enable_signaling()
406 			rdev->fence_drv[fence->ring].delayed_irq = true;  in radeon_fence_enable_signaling()
407 		radeon_fence_schedule_check(rdev, fence->ring);  in radeon_fence_enable_signaling()
410 	fence->fence_wake.flags = 0;  in radeon_fence_enable_signaling()
411 	fence->fence_wake.private = NULL;  in radeon_fence_enable_signaling()
412 	fence->fence_wake.func = radeon_fence_check_signaled;  in radeon_fence_enable_signaling()
413 	__add_wait_queue(&rdev->fence_queue, &fence->fence_wake);  in radeon_fence_enable_signaling()
419  * radeon_fence_signaled - check if a fence has signaled
421  * @fence: radeon fence object
423  * Check if the requested fence has signaled (all asics).
424  * Returns true if the fence has signaled or false if it has not.
426 bool radeon_fence_signaled(struct radeon_fence *fence)  in radeon_fence_signaled()  argument
428 	if (!fence)  in radeon_fence_signaled()
431 	if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) {  in radeon_fence_signaled()
432 		dma_fence_signal(&fence->base);  in radeon_fence_signaled()
444  * Check if the last signaled fence sequnce number is >= the requested
521  * radeon_fence_wait_timeout - wait for a fence to signal with timeout
523  * @fence: radeon fence object
526  * Wait for the requested fence to signal (all asics).
528  * (false) sleep when waiting for the fence.
533 long radeon_fence_wait_timeout(struct radeon_fence *fence, bool intr, long timeout)  in radeon_fence_wait_timeout()  argument
544 	if (WARN_ON_ONCE(!to_radeon_fence(&fence->base)))  in radeon_fence_wait_timeout()
545 		return dma_fence_wait(&fence->base, intr);  in radeon_fence_wait_timeout()
547 	seq[fence->ring] = fence->seq;  in radeon_fence_wait_timeout()
548 	r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout);  in radeon_fence_wait_timeout()
552 	dma_fence_signal(&fence->base);  in radeon_fence_wait_timeout()
557  * radeon_fence_wait - wait for a fence to signal
559  * @fence: radeon fence object
562  * Wait for the requested fence to signal (all asics).
564  * (false) sleep when waiting for the fence.
565  * Returns 0 if the fence has passed, error for all other cases.
567 int radeon_fence_wait(struct radeon_fence *fence, bool intr)  in radeon_fence_wait()  argument
569 	long r = radeon_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);  in radeon_fence_wait()
578  * radeon_fence_wait_any - wait for a fence to signal on any ring
581  * @fences: radeon fence object(s)
584  * Wait for any requested fence to signal (all asics).  Fence
588  * Returns 0 if any fence has passed, error for all other cases.
620  * radeon_fence_wait_next - wait for the next fence to signal
623  * @ring: ring index the fence is associated with
625  * Wait for the next fence on the requested ring to signal (all asics).
626  * Returns 0 if the next fence has passed, error for all other cases.
637 		 * the last emited fence  in radeon_fence_wait_next()
653  * @ring: ring index the fence is associated with
680  * radeon_fence_ref - take a ref on a fence
682  * @fence: radeon fence object
684  * Take a reference on a fence (all asics).
685  * Returns the fence.
687 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)  in radeon_fence_ref()  argument
689 	dma_fence_get(&fence->base);  in radeon_fence_ref()
690 	return fence;  in radeon_fence_ref()
694  * radeon_fence_unref - remove a ref on a fence
696  * @fence: radeon fence object
698  * Remove a reference on a fence (all asics).
700 void radeon_fence_unref(struct radeon_fence **fence)  in radeon_fence_unref()  argument
702 	struct radeon_fence *tmp = *fence;  in radeon_fence_unref()
704 	*fence = NULL;  in radeon_fence_unref()
713  * @ring: ring index the fence is associated with
724 	 * but it's ok to report slightly wrong fence count here.  in radeon_fence_count_emitted()
739  * @fence: radeon fence object
742  * Check if the fence needs to be synced against another ring
747 bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring)  in radeon_fence_need_sync()  argument
751 	if (!fence)  in radeon_fence_need_sync()
754 	if (fence->ring == dst_ring)  in radeon_fence_need_sync()
758 	fdrv = &fence->rdev->fence_drv[dst_ring];  in radeon_fence_need_sync()
759 	if (fence->seq <= fdrv->sync_seq[fence->ring])  in radeon_fence_need_sync()
768  * @fence: radeon fence object
771  * Note the sequence number at which point the fence will
774 void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring)  in radeon_fence_note_sync()  argument
779 	if (!fence)  in radeon_fence_note_sync()
782 	if (fence->ring == dst_ring)  in radeon_fence_note_sync()
786 	src = &fence->rdev->fence_drv[fence->ring];  in radeon_fence_note_sync()
787 	dst = &fence->rdev->fence_drv[dst_ring];  in radeon_fence_note_sync()
797  * radeon_fence_driver_start_ring - make the fence driver
801  * @ring: ring index to start the fence driver on
803  * Make the fence driver ready for processing (all asics).
805  * start the fence driver on the rings it has.
823 			/* put fence directly behind firmware */  in radeon_fence_driver_start_ring()
832 			dev_err(rdev->dev, "fence failed to get scratch register\n");  in radeon_fence_driver_start_ring()
843 	dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx\n",  in radeon_fence_driver_start_ring()
849  * radeon_fence_driver_init_ring - init the fence driver
853  * @ring: ring index to start the fence driver on
855  * Init the fence driver for the requested ring (all asics).
875  * radeon_fence_driver_init - init the fence driver
880  * Init the fence driver for all possible rings (all asics).
882  * start the fence driver on the rings it has using
897  * radeon_fence_driver_fini - tear down the fence driver
902  * Tear down the fence driver for all possible rings (all asics).
926  * radeon_fence_driver_force_completion - force all fence waiter to complete
931  * In case of GPU reset failure make sure no process keep waiting on fence
944  * Fence debugfs
959 		seq_printf(m, "Last signaled fence 0x%016llx\n",  in radeon_debugfs_fence_info_show()
976  * Manually trigger a gpu reset at the next fence wait.
1009 static const char *radeon_fence_get_driver_name(struct dma_fence *fence)  in radeon_fence_get_driver_name()  argument
1016 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_get_timeline_name()  local
1018 	switch (fence->ring) {  in radeon_fence_get_timeline_name()
1033 static inline bool radeon_test_signaled(struct radeon_fence *fence)  in radeon_test_signaled()  argument
1035 	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags);  in radeon_test_signaled()
1044 radeon_fence_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in radeon_fence_wait_cb()  argument
1055 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_default_wait()  local
1056 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_default_wait()
1074 		if (radeon_test_signaled(fence))  in radeon_fence_default_wait()