192 static noinline void __process_finished_items(struct rcu_pending *pending,  in __process_finished_items()  argument
201 	    __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) {  in __process_finished_items()
213 	switch ((ulong) pending->process) {  in __process_finished_items()
267 			pending->process(pending, *genradix_ptr(&objs.objs, i));  in __process_finished_items()
277 			pending->process(pending, obj);  in __process_finished_items()
283 static bool process_finished_items(struct rcu_pending *pending,  in process_finished_items()  argument
292 	if ((p->objs.nr && __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) ||  in process_finished_items()
293 	    (p->lists[0].head && __poll_state_synchronize_rcu(pending->srcu, p->lists[0].seq)) ||  in process_finished_items()
294 	    (p->lists[1].head && __poll_state_synchronize_rcu(pending->srcu, p->lists[1].seq)) ||  in process_finished_items()
296 		__process_finished_items(pending, p, flags);  in process_finished_items()
307 	struct rcu_pending *pending = p->parent;  in rcu_pending_work()  local
312 	} while (process_finished_items(pending, p, flags));  in rcu_pending_work()
403  * __rcu_pending_enqueue: enqueue a pending RCU item, to be processed (via
404  * pending->pracess) once grace period elapses.
409  * - If @ptr is NULL, we're enqueuing an item for a generic @pending with a
420 __rcu_pending_enqueue(struct rcu_pending *pending, struct rcu_head *head,  in __rcu_pending_enqueue()  argument
430 	BUG_ON((ptr != NULL) != (pending->process == RCU_PENDING_KVFREE_FN));  in __rcu_pending_enqueue()
433 	p = this_cpu_ptr(pending->p);  in __rcu_pending_enqueue()
435 	rcu_gp_poll_state_t seq = __get_state_synchronize_rcu(pending->srcu);  in __rcu_pending_enqueue()
438 	    unlikely(process_finished_items(pending, p, flags)))  in __rcu_pending_enqueue()
497 			__call_rcu(pending->srcu, &p->cb, rcu_pending_rcu_cb);  in __rcu_pending_enqueue()
499 			__start_poll_synchronize_rcu(pending->srcu);  in __rcu_pending_enqueue()
508 	if (unlikely(__poll_state_synchronize_rcu(pending->srcu, seq))) {  in __rcu_pending_enqueue()
509 		switch ((ulong) pending->process) {  in __rcu_pending_enqueue()
517 			pending->process(pending, head);  in __rcu_pending_enqueue()
524 	p = this_cpu_ptr(pending->p);  in __rcu_pending_enqueue()
529 void rcu_pending_enqueue(struct rcu_pending *pending, struct rcu_head *obj)  in rcu_pending_enqueue()  argument
531 	__rcu_pending_enqueue(pending, obj, NULL, true);  in rcu_pending_enqueue()
566 struct rcu_head *rcu_pending_dequeue(struct rcu_pending *pending)  in rcu_pending_dequeue()  argument
568 	return rcu_pending_pcpu_dequeue(raw_cpu_ptr(pending->p));  in rcu_pending_dequeue()
571 struct rcu_head *rcu_pending_dequeue_from_all(struct rcu_pending *pending)  in rcu_pending_dequeue_from_all()  argument
573 	struct rcu_head *ret = rcu_pending_dequeue(pending);  in rcu_pending_dequeue_from_all()
580 		ret = rcu_pending_pcpu_dequeue(per_cpu_ptr(pending->p, cpu));  in rcu_pending_dequeue_from_all()
587 static bool rcu_pending_has_pending_or_armed(struct rcu_pending *pending)  in rcu_pending_has_pending_or_armed()  argument
591 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_has_pending_or_armed()
603 void rcu_pending_exit(struct rcu_pending *pending)  in rcu_pending_exit()  argument
607 	if (!pending->p)  in rcu_pending_exit()
610 	while (rcu_pending_has_pending_or_armed(pending)) {  in rcu_pending_exit()
611 		__rcu_barrier(pending->srcu);  in rcu_pending_exit()
614 			struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_exit()
620 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_exit()
625 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_exit()
632 	free_percpu(pending->p);  in rcu_pending_exit()
638  * @pending:	Object to init
644 int rcu_pending_init(struct rcu_pending *pending,  in rcu_pending_init()  argument
648 	pending->p = alloc_percpu(struct rcu_pending_pcpu);  in rcu_pending_init()
649 	if (!pending->p)  in rcu_pending_init()
654 		struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu);  in rcu_pending_init()
655 		p->parent	= pending;  in rcu_pending_init()
662 	pending->srcu = srcu;  in rcu_pending_init()
663 	pending->process = process;  in rcu_pending_init()