3  * Parts ported from amdgpu (fence wait code).
41  *  - Import and export a syncobj's underlying fence to/from a sync file
42  *  - Reset a syncobj (set its fence to NULL)
43  *  - Signal a syncobj (set a trivially signaled fence)
44  *  - Wait for a syncobj's fence to appear and be signaled
57  * to an already signaled fence depending on whether the
63  * the syncobj, the syncobj's fence is replaced with a fence which will be
67  * struct &dma_fence_chain pointing to the DRM driver's fence and also
68  * pointing to the previous fence that was in the syncobj. The new struct
69  * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
71  * fence previously in the syncobj.
74  * time the work is enqueued, it waits on the syncobj's fence before
75  * submitting the work to hardware. That fence is either :
77  *    - The syncobj's current fence if the syncobj is considered as a binary
82  * If the syncobj's fence is NULL or not present in the syncobj's timeline,
85  * With binary syncobj, all manipulation of the syncobjs's fence happens in
86  * terms of the current fence at the time the ioctl is called by userspace
91  * setting its pointer to a fence which is already signaled.
93  * With a timeline syncobj, all manipulation of the synobj's fence happens in
104  * syncobj's fence when signaling).
114  * Otherwise, it returns once at least one syncobj fence has been signaled
115  * and the index of a signaled fence is written back to the client.
118  * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
120  * fence and then wait on that fence.
122  * syncobjs in the array has a NULL fence, -EINVAL will be returned.
123  * Assuming the syncobj starts off with a NULL fence, this allows a client
127  * This requirement is inherited from the Vulkan fence API.
130  * a fence deadline hint on the backing fences before waiting, to provide the
131  * fence signaler with an appropriate sense of urgency.  The deadline is
139  * fence to materialize on the timeline without waiting for the fence to be
172  * import/export the syncobj's current fence from/to a &sync_file.
174  * sycnobj's fence at the time of export and any later signal or reset
176  * When a sync file is imported into a syncobj, the syncobj's fence is set
177  * to the fence wrapped by that sync file.
192  * point 0 to mean take/replace the fence in the syncobj.
196 #include <linux/dma-fence-unwrap.h>
217 	struct dma_fence *fence;  member
227 	struct dma_fence *fence;  member
268 	struct dma_fence *fence;  in drm_syncobj_fence_add_wait()  local
270 	if (wait->fence)  in drm_syncobj_fence_add_wait()
274 	/* We've already tried once to get a fence and failed.  Now that we  in drm_syncobj_fence_add_wait()
276 	 * callback when a fence has already been set.  in drm_syncobj_fence_add_wait()
278 	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));  in drm_syncobj_fence_add_wait()
279 	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {  in drm_syncobj_fence_add_wait()
280 		dma_fence_put(fence);  in drm_syncobj_fence_add_wait()
282 	} else if (!fence) {  in drm_syncobj_fence_add_wait()
283 		wait->fence = dma_fence_get_stub();  in drm_syncobj_fence_add_wait()
285 		wait->fence = fence;  in drm_syncobj_fence_add_wait()
305 	dma_fence_put(entry->fence);  in syncobj_eventfd_entry_free()
327  * @fence: fence to encapsulate in the chain node
334 			   struct dma_fence *fence,  in drm_syncobj_add_point()  argument
341 	dma_fence_get(fence);  in drm_syncobj_add_point()
349 	dma_fence_chain_init(chain, prev, fence, point);  in drm_syncobj_add_point()
350 	rcu_assign_pointer(syncobj->fence, &chain->base);  in drm_syncobj_add_point()
359 	dma_fence_chain_for_each(fence, prev);  in drm_syncobj_add_point()
365  * drm_syncobj_replace_fence - replace fence in a sync object.
366  * @syncobj: Sync object to replace fence in
367  * @fence: fence to install in sync file.
369  * This replaces the fence on a sync object.
372 			       struct dma_fence *fence)  in drm_syncobj_replace_fence()  argument
378 	if (fence)  in drm_syncobj_replace_fence()
379 		dma_fence_get(fence);  in drm_syncobj_replace_fence()
383 	old_fence = rcu_dereference_protected(syncobj->fence,  in drm_syncobj_replace_fence()
385 	rcu_assign_pointer(syncobj->fence, fence);  in drm_syncobj_replace_fence()
387 	if (fence != old_fence) {  in drm_syncobj_replace_fence()
401  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
402  * @syncobj: sync object to assign the fence on
404  * Assign a already signaled stub fence to the sync object.
408 	struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());  in drm_syncobj_assign_null_handle()  local
410 	if (!fence)  in drm_syncobj_assign_null_handle()
413 	drm_syncobj_replace_fence(syncobj, fence);  in drm_syncobj_assign_null_handle()
414 	dma_fence_put(fence);  in drm_syncobj_assign_null_handle()
421  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
426  * @fence: out parameter for the fence
431  * Returns 0 on success or a negative error value on failure. On success @fence
432  * contains a reference to the fence, which must be released by calling
437 			   struct dma_fence **fence)  in drm_syncobj_find_fence()  argument
459 	*fence = drm_syncobj_fence_get(syncobj);  in drm_syncobj_find_fence()
461 	if (*fence) {  in drm_syncobj_find_fence()
462 		ret = dma_fence_chain_find_seqno(fence, point);  in drm_syncobj_find_fence()
466 			 * fence. To make sure the recipient gets  in drm_syncobj_find_fence()
467 			 * signalled, use a new fence instead.  in drm_syncobj_find_fence()
469 			if (!*fence)  in drm_syncobj_find_fence()
470 				*fence = dma_fence_get_stub();  in drm_syncobj_find_fence()
474 		dma_fence_put(*fence);  in drm_syncobj_find_fence()
489 		if (wait.fence) {  in drm_syncobj_find_fence()
507 	*fence = wait.fence;  in drm_syncobj_find_fence()
545  * @fence: if non-NULL, the syncobj will represent this fence
554 		       struct dma_fence *fence)  in drm_syncobj_create()  argument
576 	if (fence)  in drm_syncobj_create()
577 		drm_syncobj_replace_fence(syncobj, fence);  in drm_syncobj_create()
746 	struct dma_fence *fence = sync_file_get_fence(fd);  in drm_syncobj_import_sync_file_fence()  local
749 	if (!fence)  in drm_syncobj_import_sync_file_fence()
754 		dma_fence_put(fence);  in drm_syncobj_import_sync_file_fence()
758 	drm_syncobj_replace_fence(syncobj, fence);  in drm_syncobj_import_sync_file_fence()
759 	dma_fence_put(fence);  in drm_syncobj_import_sync_file_fence()
768 	struct dma_fence *fence;  in drm_syncobj_export_sync_file()  local
775 	ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);  in drm_syncobj_export_sync_file()
779 	sync_file = sync_file_create(fence);  in drm_syncobj_export_sync_file()
781 	dma_fence_put(fence);  in drm_syncobj_export_sync_file()
920 	struct dma_fence *fence, *tmp;  in drm_syncobj_transfer_to_timeline()  local
934 	fence = dma_fence_unwrap_merge(tmp);  in drm_syncobj_transfer_to_timeline()
936 	if (!fence) {  in drm_syncobj_transfer_to_timeline()
947 	drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);  in drm_syncobj_transfer_to_timeline()
949 	dma_fence_put(fence);  in drm_syncobj_transfer_to_timeline()
961 	struct dma_fence *fence;  in drm_syncobj_transfer_to_binary()  local
968 				     args->src_point, args->flags, &fence);  in drm_syncobj_transfer_to_binary()
971 	drm_syncobj_replace_fence(binary_syncobj, fence);  in drm_syncobj_transfer_to_binary()
972 	dma_fence_put(fence);  in drm_syncobj_transfer_to_binary()
999 static void syncobj_wait_fence_func(struct dma_fence *fence,  in syncobj_wait_fence_func()  argument
1011 	struct dma_fence *fence;  in syncobj_wait_syncobj_func()  local
1014 	fence = rcu_dereference_protected(syncobj->fence,  in syncobj_wait_syncobj_func()
1016 	dma_fence_get(fence);  in syncobj_wait_syncobj_func()
1017 	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {  in syncobj_wait_syncobj_func()
1018 		dma_fence_put(fence);  in syncobj_wait_syncobj_func()
1020 	} else if (!fence) {  in syncobj_wait_syncobj_func()
1021 		wait->fence = dma_fence_get_stub();  in syncobj_wait_syncobj_func()
1023 		wait->fence = fence;  in syncobj_wait_syncobj_func()
1039 	struct dma_fence *fence;  in drm_syncobj_array_wait_timeout()  local
1069 	 * a syncobj with a missing fence and then never have the chance of  in drm_syncobj_array_wait_timeout()
1074 		struct dma_fence *fence;  in drm_syncobj_array_wait_timeout()  local
1078 		fence = drm_syncobj_fence_get(syncobjs[i]);  in drm_syncobj_array_wait_timeout()
1079 		if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {  in drm_syncobj_array_wait_timeout()
1080 			dma_fence_put(fence);  in drm_syncobj_array_wait_timeout()
1090 		if (fence)  in drm_syncobj_array_wait_timeout()
1091 			entries[i].fence = fence;  in drm_syncobj_array_wait_timeout()
1093 			entries[i].fence = dma_fence_get_stub();  in drm_syncobj_array_wait_timeout()
1096 		    dma_fence_is_signaled(entries[i].fence)) {  in drm_syncobj_array_wait_timeout()
1110 	 * fence is signaled prior to fence->ops->enable_signaling() being  in drm_syncobj_array_wait_timeout()
1123 			fence = entries[i].fence;  in drm_syncobj_array_wait_timeout()
1124 			if (!fence)  in drm_syncobj_array_wait_timeout()
1126 			dma_fence_set_deadline(fence, *deadline);  in drm_syncobj_array_wait_timeout()
1135 			fence = entries[i].fence;  in drm_syncobj_array_wait_timeout()
1136 			if (!fence)  in drm_syncobj_array_wait_timeout()
1140 			    dma_fence_is_signaled(fence) ||  in drm_syncobj_array_wait_timeout()
1142 			     dma_fence_add_callback(fence,  in drm_syncobj_array_wait_timeout()
1145 				/* The fence has been signaled */  in drm_syncobj_array_wait_timeout()
1179 			dma_fence_remove_callback(entries[i].fence,  in drm_syncobj_array_wait_timeout()
1181 		dma_fence_put(entries[i].fence);  in drm_syncobj_array_wait_timeout()
1404 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,  in syncobj_eventfd_entry_fence_func()  argument
1419 	struct dma_fence *fence;  in syncobj_eventfd_entry_func()  local
1422 	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));  in syncobj_eventfd_entry_func()
1423 	if (!fence)  in syncobj_eventfd_entry_func()
1426 	ret = dma_fence_chain_find_seqno(&fence, entry->point);  in syncobj_eventfd_entry_func()
1429 		dma_fence_put(fence);  in syncobj_eventfd_entry_func()
1431 	} else if (!fence) {  in syncobj_eventfd_entry_func()
1432 		/* If dma_fence_chain_find_seqno returns 0 but sets the fence  in syncobj_eventfd_entry_func()
1434 		 * later seqno has already been submitted. Assign a stub fence  in syncobj_eventfd_entry_func()
1437 		fence = dma_fence_get_stub();  in syncobj_eventfd_entry_func()
1441 	entry->fence = fence;  in syncobj_eventfd_entry_func()
1447 		ret = dma_fence_add_callback(fence, &entry->fence_cb,  in syncobj_eventfd_entry_func()
1633 		struct dma_fence *fence = dma_fence_get_stub();  in drm_syncobj_timeline_signal_ioctl()  local
1636 				      fence, points[i]);  in drm_syncobj_timeline_signal_ioctl()
1637 		dma_fence_put(fence);  in drm_syncobj_timeline_signal_ioctl()
1676 		struct dma_fence *fence;  in drm_syncobj_query_ioctl()  local
1679 		fence = drm_syncobj_fence_get(syncobjs[i]);  in drm_syncobj_query_ioctl()
1680 		chain = to_dma_fence_chain(fence);  in drm_syncobj_query_ioctl()
1683 				dma_fence_get(fence);  in drm_syncobj_query_ioctl()
1687 				point = fence->seqno;  in drm_syncobj_query_ioctl()
1689 				dma_fence_chain_for_each(iter, fence) {  in drm_syncobj_query_ioctl()
1690 					if (iter->context != fence->context) {  in drm_syncobj_query_ioctl()
1707 		dma_fence_put(fence);  in drm_syncobj_query_ioctl()