3  * Fence mechanism for dma-buf and to allow for asynchronous dma access
16 #include <linux/dma-fence.h>
31  * fence context counter: each execution context should have its own
32  * fence context, this allows checking if fences belong to the same
45  * A fence is initialized using dma_fence_init() and completed using
60  *   fence to be updated.
69  * DOC: fence cross-driver contract
95  *   This means any code required for fence completion cannot acquire a
100  *   callbacks. This means any code required for fence completion cannot
105  *   for fence completion cannot allocate memory with GFP_NOFS or GFP_NOIO.
114 static const char *dma_fence_stub_get_name(struct dma_fence *fence)  in dma_fence_stub_get_name()  argument
125  * dma_fence_get_stub - return a signaled fence
127  * Return a stub fence which is already signaled. The fence's
152  * dma_fence_allocate_private_stub - return a private, signaled fence
153  * @timestamp: timestamp when the fence was signaled
155  * Return a newly allocated and signaled stub fence.
159 	struct dma_fence *fence;  in dma_fence_allocate_private_stub()  local
161 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);  in dma_fence_allocate_private_stub()
162 	if (fence == NULL)  in dma_fence_allocate_private_stub()
165 	dma_fence_init(fence,  in dma_fence_allocate_private_stub()
171 		&fence->flags);  in dma_fence_allocate_private_stub()
173 	dma_fence_signal_timestamp(fence, timestamp);  in dma_fence_allocate_private_stub()
175 	return fence;  in dma_fence_allocate_private_stub()
180  * dma_fence_context_alloc - allocate an array of fence contexts
183  * This function will return the first index of the number of fence contexts
184  * allocated.  The fence context is used for setting &dma_fence.context to a
195  * DOC: fence signalling annotation
235  * prevents it from signalling the fence the previous thread is stuck waiting
256  *   point where a fence is accessible to other threads, to the point where
289  * dma_fence_begin_signalling - begin a critical DMA fence signalling section
320  * dma_fence_end_signalling - end a critical DMA fence signalling section
350  * dma_fence_signal_timestamp_locked - signal completion of a fence
351  * @fence: the fence to signal
352  * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
354  * Signal completion for software callbacks on a fence, this will unblock
356  * dma_fence_add_callback(). Can be called multiple times, but since a fence
358  * only be effective the first time. Set the timestamp provided as the fence
364  * Returns 0 on success and a negative error value when @fence has been
367 int dma_fence_signal_timestamp_locked(struct dma_fence *fence,  in dma_fence_signal_timestamp_locked()  argument
373 	lockdep_assert_held(fence->lock);  in dma_fence_signal_timestamp_locked()
376 				      &fence->flags)))  in dma_fence_signal_timestamp_locked()
380 	list_replace(&fence->cb_list, &cb_list);  in dma_fence_signal_timestamp_locked()
382 	fence->timestamp = timestamp;  in dma_fence_signal_timestamp_locked()
383 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_timestamp_locked()
384 	trace_dma_fence_signaled(fence);  in dma_fence_signal_timestamp_locked()
388 		cur->func(fence, cur);  in dma_fence_signal_timestamp_locked()
396  * dma_fence_signal_timestamp - signal completion of a fence
397  * @fence: the fence to signal
398  * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
400  * Signal completion for software callbacks on a fence, this will unblock
402  * dma_fence_add_callback(). Can be called multiple times, but since a fence
404  * only be effective the first time. Set the timestamp provided as the fence
407  * Returns 0 on success and a negative error value when @fence has been
410 int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)  in dma_fence_signal_timestamp()  argument
415 	if (WARN_ON(!fence))  in dma_fence_signal_timestamp()
418 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal_timestamp()
419 	ret = dma_fence_signal_timestamp_locked(fence, timestamp);  in dma_fence_signal_timestamp()
420 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal_timestamp()
427  * dma_fence_signal_locked - signal completion of a fence
428  * @fence: the fence to signal
430  * Signal completion for software callbacks on a fence, this will unblock
432  * dma_fence_add_callback(). Can be called multiple times, but since a fence
439  * Returns 0 on success and a negative error value when @fence has been
442 int dma_fence_signal_locked(struct dma_fence *fence)  in dma_fence_signal_locked()  argument
444 	return dma_fence_signal_timestamp_locked(fence, ktime_get());  in dma_fence_signal_locked()
449  * dma_fence_signal - signal completion of a fence
450  * @fence: the fence to signal
452  * Signal completion for software callbacks on a fence, this will unblock
454  * dma_fence_add_callback(). Can be called multiple times, but since a fence
458  * Returns 0 on success and a negative error value when @fence has been
461 int dma_fence_signal(struct dma_fence *fence)  in dma_fence_signal()  argument
467 	if (WARN_ON(!fence))  in dma_fence_signal()
472 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
473 	ret = dma_fence_signal_timestamp_locked(fence, ktime_get());  in dma_fence_signal()
474 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
483  * dma_fence_wait_timeout - sleep until the fence gets signaled
485  * @fence: the fence to wait on
493  * Performs a synchronous wait on this fence. It is assumed the caller
495  * holds a reference to the fence, otherwise the fence might be
501 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_wait_timeout()  argument
512 	dma_fence_enable_sw_signaling(fence);  in dma_fence_wait_timeout()
514 	trace_dma_fence_wait_start(fence);  in dma_fence_wait_timeout()
515 	if (fence->ops->wait)  in dma_fence_wait_timeout()
516 		ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
518 		ret = dma_fence_default_wait(fence, intr, timeout);  in dma_fence_wait_timeout()
519 	trace_dma_fence_wait_end(fence);  in dma_fence_wait_timeout()
533 	struct dma_fence *fence =  in dma_fence_release()  local
536 	trace_dma_fence_destroy(fence);  in dma_fence_release()
538 	if (WARN(!list_empty(&fence->cb_list) &&  in dma_fence_release()
539 		 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),  in dma_fence_release()
540 		 "Fence %s:%s:%llx:%llx released with pending signals!\n",  in dma_fence_release()
541 		 fence->ops->get_driver_name(fence),  in dma_fence_release()
542 		 fence->ops->get_timeline_name(fence),  in dma_fence_release()
543 		 fence->context, fence->seqno)) {  in dma_fence_release()
553 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_release()
554 		fence->error = -EDEADLK;  in dma_fence_release()
555 		dma_fence_signal_locked(fence);  in dma_fence_release()
556 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_release()
559 	if (fence->ops->release)  in dma_fence_release()
560 		fence->ops->release(fence);  in dma_fence_release()
562 		dma_fence_free(fence);  in dma_fence_release()
568  * @fence: fence to release
571  * kfree_rcu() on @fence.
573 void dma_fence_free(struct dma_fence *fence)  in dma_fence_free()  argument
575 	kfree_rcu(fence, rcu);  in dma_fence_free()
579 static bool __dma_fence_enable_signaling(struct dma_fence *fence)  in __dma_fence_enable_signaling()  argument
583 	lockdep_assert_held(fence->lock);  in __dma_fence_enable_signaling()
586 				   &fence->flags);  in __dma_fence_enable_signaling()
588 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in __dma_fence_enable_signaling()
591 	if (!was_set && fence->ops->enable_signaling) {  in __dma_fence_enable_signaling()
592 		trace_dma_fence_enable_signal(fence);  in __dma_fence_enable_signaling()
594 		if (!fence->ops->enable_signaling(fence)) {  in __dma_fence_enable_signaling()
595 			dma_fence_signal_locked(fence);  in __dma_fence_enable_signaling()
604  * dma_fence_enable_sw_signaling - enable signaling on fence
605  * @fence: the fence to enable
607  * This will request for sw signaling to be enabled, to make the fence
611 void dma_fence_enable_sw_signaling(struct dma_fence *fence)  in dma_fence_enable_sw_signaling()  argument
615 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
616 	__dma_fence_enable_signaling(fence);  in dma_fence_enable_sw_signaling()
617 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
622  * dma_fence_add_callback - add a callback to be called when the fence
624  * @fence: the fence to wait on
628  * Add a software callback to the fence. The caller should keep a reference to
629  * the fence.
633  * to a fence, but a callback can only be registered to one fence at a time.
635  * If fence is already signaled, this function will return -ENOENT (and
640  * Returns 0 in case of success, -ENOENT if the fence is already signaled
643 int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,  in dma_fence_add_callback()  argument
649 	if (WARN_ON(!fence || !func))  in dma_fence_add_callback()
652 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
657 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
659 	if (__dma_fence_enable_signaling(fence)) {  in dma_fence_add_callback()
661 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
667 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
675  * @fence: the dma_fence to query
678  * condition on a signaled fence. See dma_fence_get_status_locked() for more
681  * Returns 0 if the fence has not yet been signaled, 1 if the fence has
683  * if the fence has been completed in err.
685 int dma_fence_get_status(struct dma_fence *fence)  in dma_fence_get_status()  argument
690 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
691 	status = dma_fence_get_status_locked(fence);  in dma_fence_get_status()
692 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
700  * @fence: the fence to wait on
703  * Remove a previously queued callback from the fence. This function returns
704  * true if the callback is successfully removed, or false if the fence has
711  * with a reference held to the fence.
713  * Behaviour is undefined if @cb has not been added to @fence using
717 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_remove_callback()  argument
722 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
728 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
740 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_default_wait_cb()  argument
749  * dma_fence_default_wait - default sleep until the fence gets signaled
751  * @fence: the fence to wait on
757  * returned if the fence is already signaled for consistency with other
761 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_default_wait()  argument
767 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
769 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
784 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
786 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
791 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
795 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
805 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
817 		struct dma_fence *fence = fences[i];  in dma_fence_test_signaled_any()  local
818 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
828  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
834  * @idx: used to store the first signaled fence index, meaningful only on
837  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
841  * Synchronous waits for the first fence in the array to be signaled. The
843  * fence might be freed before return, resulting in undefined behavior.
876 		struct dma_fence *fence = fences[i];  in dma_fence_wait_any_timeout()  local
879 		if (dma_fence_add_callback(fence, &cb[i].base,  in dma_fence_wait_any_timeout()
881 			/* This fence is already signaled */  in dma_fence_wait_any_timeout()
946  * hint (Ie. forward progress cannot be made until this fence is signaled).
948  * Multiple deadlines may be set on a given fence, even in parallel.  See the
951  * The deadline hint is just that, a hint.  The driver that created the fence
957  * dma_fence_set_deadline - set desired fence-wait deadline hint
958  * @fence:    the fence that is to be waited on
959  * @deadline: the time by which the waiter hopes for the fence to be
962  * Give the fence signaler a hint about an upcoming deadline, such as
963  * vblank, by which point the waiter would prefer the fence to be
964  * signaled by.  This is intended to give feedback to the fence signaler
966  * if a periodic vblank deadline is approaching but the fence is not
969 void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)  in dma_fence_set_deadline()  argument
971 	if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))  in dma_fence_set_deadline()
972 		fence->ops->set_deadline(fence, deadline);  in dma_fence_set_deadline()
977  * dma_fence_describe - Dump fence description into seq_file
978  * @fence: the fence to describe
981  * Dump a textual description of the fence and it's state into the seq_file.
983 void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq)  in dma_fence_describe()  argument
986 		   fence->ops->get_driver_name(fence),  in dma_fence_describe()
987 		   fence->ops->get_timeline_name(fence), fence->seqno,  in dma_fence_describe()
988 		   dma_fence_is_signaled(fence) ? "" : "un");  in dma_fence_describe()
993  * dma_fence_init - Initialize a custom fence.
994  * @fence: the fence to initialize
995  * @ops: the dma_fence_ops for operations on this fence
996  * @lock: the irqsafe spinlock to use for locking this fence
997  * @context: the execution context this fence is run on
1000  * Initializes an allocated fence, the caller doesn't have to keep its
1001  * refcount after committing with this fence, but it will need to hold a
1005  * to check which fence is later by simply using dma_fence_later().
1008 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,  in dma_fence_init()  argument
1014 	kref_init(&fence->refcount);  in dma_fence_init()
1015 	fence->ops = ops;  in dma_fence_init()
1016 	INIT_LIST_HEAD(&fence->cb_list);  in dma_fence_init()
1017 	fence->lock = lock;  in dma_fence_init()
1018 	fence->context = context;  in dma_fence_init()
1019 	fence->seqno = seqno;  in dma_fence_init()
1020 	fence->flags = 0UL;  in dma_fence_init()
1021 	fence->error = 0;  in dma_fence_init()
1023 	trace_dma_fence_init(fence);  in dma_fence_init()