8  * RCU-protected list version
10 #include <linux/list.h>
15  * @list: list to be initialized
18  * cleanup tasks, when readers have no access to the list being initialized.
19  * However, if the list being initialized is visible to readers, you
22 static inline void INIT_LIST_HEAD_RCU(struct list_head *list)  in INIT_LIST_HEAD_RCU()  argument
24 	WRITE_ONCE(list->next, list);  in INIT_LIST_HEAD_RCU()
25 	WRITE_ONCE(list->prev, list);  in INIT_LIST_HEAD_RCU()
32 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))  argument
37  * Any list traversed with list_bidir_prev_rcu() must never use
43 #define list_bidir_prev_rcu(list) (*((struct list_head __rcu **)(&(list)->prev)))  argument
46  * list_tail_rcu - returns the prev pointer of the head of the list
47  * @head: the head of the list
49  * Note: This should only be used with the list header, and even then
51  * list header.
56  * Check during list traversal that we are within an RCU reader
66 			 "RCU-list traversed in non-reader section!");	\
72 		"RCU-list traversed without holding the required lock!");\
84  * This is only for internal list manipulation where we know
100  * list_add_rcu - add a new entry to rcu-protected list
102  * @head: list head to add it after
109  * with another list-mutation primitive, such as list_add_rcu()
110  * or list_del_rcu(), running on this same list.
112  * the _rcu list-traversal primitives, such as
121  * list_add_tail_rcu - add a new entry to rcu-protected list
123  * @head: list head to add it before
130  * with another list-mutation primitive, such as list_add_tail_rcu()
131  * or list_del_rcu(), running on this same list.
133  * the _rcu list-traversal primitives, such as
143  * list_del_rcu - deletes entry from list without re-initialization
144  * @entry: the element to delete from the list.
151  * pointers that may still be used for walking the list.
155  * with another list-mutation primitive, such as list_del_rcu()
156  * or list_add_rcu(), running on this same list.
158  * the _rcu list-traversal primitives, such as
173  * list_bidir_del_rcu - deletes entry from list without re-initialization
174  * @entry: the element to delete from the list.
183  * walking the list.
186  * holding appropriate locks) to avoid racing with another list-mutation
188  * this same list. However, it is perfectly legal to run concurrently
189  * with the _rcu list-traversal primitives, such as
193  * the same list.
206  * hlist_del_init_rcu - deletes entry from hash list with re-initialization
207  * @n: the element to delete from the hash list.
211  * must know if the list entry is still hashed or already unhashed.
214  * that may still be used for walking the hash list and we can only
220  * list-mutation primitive, such as hlist_add_head_rcu() or
221  * hlist_del_rcu(), running on this same list.  However, it is
222  * perfectly legal to run concurrently with the _rcu list-traversal
255  * __list_splice_init_rcu - join an RCU-protected list into an existing list.
256  * @list:	the RCU-protected list to splice
257  * @prev:	points to the last element of the existing list
258  * @next:	points to the first element of the existing list
261  * The list pointed to by @prev and @next can be RCU-read traversed
267  * any other updates to the existing list.  In principle, it is possible to
268  * modify the list as soon as sync() begins execution. If this sort of thing
273 static inline void __list_splice_init_rcu(struct list_head *list,  in __list_splice_init_rcu()  argument
278 	struct list_head *first = list->next;  in __list_splice_init_rcu()
279 	struct list_head *last = list->prev;  in __list_splice_init_rcu()
282 	 * "first" and "last" tracking list, so initialize it.  RCU readers  in __list_splice_init_rcu()
283 	 * have access to this list, so we must use INIT_LIST_HEAD_RCU()  in __list_splice_init_rcu()
287 	INIT_LIST_HEAD_RCU(list);  in __list_splice_init_rcu()
290 	 * At this point, the list body still points to the source list.  in __list_splice_init_rcu()
291 	 * Wait for any readers to finish using the list before splicing  in __list_splice_init_rcu()
292 	 * the list body into the new list.  Any new readers will see  in __list_splice_init_rcu()
293 	 * an empty list.  in __list_splice_init_rcu()
301 	 * Readers are finished with the source list, so perform splice.  in __list_splice_init_rcu()
302 	 * The order is important if the new list is global and accessible  in __list_splice_init_rcu()
315  * list_splice_init_rcu - splice an RCU-protected list into an existing list,
317  * @list:	the RCU-protected list to splice
318  * @head:	the place in the existing list to splice the first list into
321 static inline void list_splice_init_rcu(struct list_head *list,  in list_splice_init_rcu()  argument
325 	if (!list_empty(list))  in list_splice_init_rcu()
326 		__list_splice_init_rcu(list, head, head->next, sync);  in list_splice_init_rcu()
330  * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
331  *                             list, designed for queues.
332  * @list:	the RCU-protected list to splice
333  * @head:	the place in the existing list to splice the first list into
336 static inline void list_splice_tail_init_rcu(struct list_head *list,  in list_splice_tail_init_rcu()  argument
340 	if (!list_empty(list))  in list_splice_tail_init_rcu()
341 		__list_splice_init_rcu(list, head->prev, head, sync);  in list_splice_tail_init_rcu()
350  * This primitive may safely run concurrently with the _rcu list-mutation
366  * The list might be non-empty when list_empty_rcu() checks it, but it
376  * list head.  However, it neither dereferences this pointer nor provides
387  * list_first_or_null_rcu - get the first element from a list
388  * @ptr:        the list head to take the element from.
392  * Note that if the list is empty, it returns NULL.
394  * This primitive may safely run concurrently with the _rcu list-mutation
405  * list_next_or_null_rcu - get the next element from a list
406  * @head:	the head for the list.
407  * @ptr:        the list head to take the next element from.
411  * Note that if the ptr is at the end of the list, NULL is returned.
413  * This primitive may safely run concurrently with the _rcu list-mutation
426  * list_for_each_entry_rcu	-	iterate over rcu list of given type
428  * @head:	the head for your list.
432  * This list-traversal primitive may safely run concurrently with
433  * the _rcu list-mutation primitives such as list_add_rcu()
443  * list_for_each_entry_srcu	-	iterate over rcu list of given type
445  * @head:	the head for your list.
447  * @cond:	lockdep expression for the lock required to traverse the list.
449  * This list-traversal primitive may safely run concurrently with
450  * the _rcu list-mutation primitives such as list_add_rcu()
468  * list-mutation primitives such as list_add_rcu(), but requires some
471  * cannot be invoked.  Another example is when items are added to the list,
478  * list_for_each_entry_lockless - iterate over rcu list of given type
480  * @head:	the head for your list.
484  * list-mutation primitives such as list_add_rcu(), but requires some
487  * cannot be invoked.  Another example is when items are added to the list,
496  * list_for_each_entry_continue_rcu - continue iteration over list of given type
498  * @head:	the head for your list.
501  * Continue to iterate over list of given type, continuing after
502  * the current position which must have been in the list when the RCU read
505  * previous walk of the list in the same RCU read-side critical section, or
507  * to keep the node alive *and* in the list.
519  * list_for_each_entry_from_rcu - iterate over a list from current point
521  * @head:	the head for your list.
524  * Iterate over the tail of a list starting from a given position,
525  * which must have been in the list when the RCU read lock was taken.
527  * previous walk of the list in the same RCU read-side critical section, or
529  * to keep the node alive *and* in the list.
540  * hlist_del_rcu - deletes entry from hash list without re-initialization
541  * @n: the element to delete from the hash list.
548  * pointers that may still be used for walking the hash list.
552  * with another list-mutation primitive, such as hlist_add_head_rcu()
553  * or hlist_del_rcu(), running on this same list.
555  * the _rcu list-traversal primitives, such as
616  * @n: the element to add to the hash list.
617  * @h: the list to add to.
625  * with another list-mutation primitive, such as hlist_add_head_rcu()
626  * or hlist_del_rcu(), running on this same list.
628  * the _rcu list-traversal primitives, such as
631  * list-traversal primitive must be guarded by rcu_read_lock().
647  * @n: the element to add to the hash list.
648  * @h: the list to add to.
656  * with another list-mutation primitive, such as hlist_add_head_rcu()
657  * or hlist_del_rcu(), running on this same list.
659  * the _rcu list-traversal primitives, such as
662  * list-traversal primitive must be guarded by rcu_read_lock().
684  * @n: the new element to add to the hash list.
693  * with another list-mutation primitive, such as hlist_add_head_rcu()
694  * or hlist_del_rcu(), running on this same list.
696  * the _rcu list-traversal primitives, such as
711  * @n: the new element to add to the hash list.
720  * with another list-mutation primitive, such as hlist_add_head_rcu()
721  * or hlist_del_rcu(), running on this same list.
723  * the _rcu list-traversal primitives, such as
743  * hlist_for_each_entry_rcu - iterate over rcu list of given type
745  * @head:	the head for your list.
749  * This list-traversal primitive may safely run concurrently with
750  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
762  * hlist_for_each_entry_srcu - iterate over rcu list of given type
764  * @head:	the head for your list.
766  * @cond:	lockdep expression for the lock required to traverse the list.
768  * This list-traversal primitive may safely run concurrently with
769  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
783  * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
785  * @head:	the head for your list.
788  * This list-traversal primitive may safely run concurrently with
789  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
803  * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
805  * @head:	the head for your list.
808  * This list-traversal primitive may safely run concurrently with
809  * the _rcu list-mutation primitives such as hlist_add_head_rcu()