Lines Matching +full:wait +full:- +full:free +full:- +full:us
1 // SPDX-License-Identifier: GPL-2.0-or-later
25 #include <linux/wait.h>
48 * 2) ep->mtx (mutex)
49 * 3) ep->lock (rwlock)
52 * We need a rwlock (ep->lock) because we manipulate objects
58 * we need a lock that will allow us to sleep. This lock is a
59 * mutex (ep->mtx). It is acquired during the event transfer loop,
68 * It is necessary to acquire multiple "ep->mtx"es at once in the
71 * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
72 * before e2->mtx). Since we disallow cycles of epoll file
73 * descriptors, this ensures that the mutexes are well-ordered. In
77 * It is possible to drop the "ep->mtx" and to use the global
78 * mutex "epnested_mutex" (together with "ep->lock") to have it working,
79 * but having "ep->mtx" will make the interface more scalable.
81 * normal operations the epoll private "ep->mtx" will guarantee
98 #define EP_UNACTIVE_PTR ((void *) -1L)
107 /* Wait structure used by the poll hooks */
116 * Wait queue item that will be linked to the target file wait
119 wait_queue_entry_t wait; member
121 /* The wait queue head that linked the "wait" wait queue item */
135 /* Used to free the struct epitem */
143 * Works together "struct eventpoll"->ovflist in keeping the
152 * Protected by file->f_lock, true for to-be-released epitem already
154 * eventpoll->refcount orchestrates "struct eventpoll" disposal
158 /* List containing poll wait queues */
188 /* Wait queue used by sys_epoll_wait() */
191 /* Wait queue used by file->poll() */
206 * holding ->lock.
223 * usage count, used together with epitem->dying to
292 head = container_of(file->f_ep, struct epitems_head, epitems); in list_file()
293 if (!head->next) { in list_file()
294 head->next = tfile_check_list; in list_file()
302 struct hlist_node *p = rcu_dereference(hlist_first_rcu(&head->epitems)); in unlist_file()
305 spin_lock(&epi->ffd.file->f_lock); in unlist_file()
306 if (!hlist_empty(&head->epitems)) in unlist_file()
308 head->next = NULL; in unlist_file()
309 spin_unlock(&epi->ffd.file->f_lock); in unlist_file()
345 return f->f_op == &eventpoll_fops; in is_file_epoll()
352 ffd->file = file; in ep_set_ffd()
353 ffd->fd = fd; in ep_set_ffd()
360 return (p1->file > p2->file ? +1: in ep_cmp_ffd()
361 (p1->file < p2->file ? -1 : p1->fd - p2->fd)); in ep_cmp_ffd()
364 /* Tells us if the item is currently linked */
367 return !list_empty(&epi->rdllink); in ep_is_linked()
372 return container_of(p, struct eppoll_entry, wait); in ep_pwq_from_wait()
375 /* Get the "struct epitem" from a wait queue pointer */
378 return container_of(p, struct eppoll_entry, wait)->base; in ep_item_from_wait()
382 * ep_events_available - Checks if ready events might be available.
391 return !list_empty_careful(&ep->rdllist) || in ep_events_available()
392 READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; in ep_events_available()
397 * busy_loop_ep_timeout - check if busy poll has timed out. The timeout value
399 * the system-wide global via busy_loop_timeout.
409 unsigned long bp_usec = READ_ONCE(ep->busy_poll_usecs); in busy_loop_ep_timeout()
423 return !!READ_ONCE(ep->busy_poll_usecs) || in ep_busy_loop_on()
424 READ_ONCE(ep->prefer_busy_poll) || in ep_busy_loop_on()
443 unsigned int napi_id = READ_ONCE(ep->napi_id); in ep_busy_loop()
444 u16 budget = READ_ONCE(ep->busy_poll_budget); in ep_busy_loop()
445 bool prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll); in ep_busy_loop()
462 ep->napi_id = 0; in ep_busy_loop()
473 struct eventpoll *ep = epi->ep; in ep_set_busy_poll_napi_id()
481 sock = sock_from_file(epi->ffd.file); in ep_set_busy_poll_napi_id()
485 sk = sock->sk; in ep_set_busy_poll_napi_id()
489 napi_id = READ_ONCE(sk->sk_napi_id); in ep_set_busy_poll_napi_id()
491 /* Non-NAPI IDs can be rejected in ep_set_busy_poll_napi_id()
495 if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id) in ep_set_busy_poll_napi_id()
499 ep->napi_id = napi_id; in ep_set_busy_poll_napi_id()
505 struct eventpoll *ep = file->private_data; in ep_eventpoll_bp_ioctl()
512 return -EFAULT; in ep_eventpoll_bp_ioctl()
516 return -EINVAL; in ep_eventpoll_bp_ioctl()
519 return -EINVAL; in ep_eventpoll_bp_ioctl()
522 return -EINVAL; in ep_eventpoll_bp_ioctl()
526 return -EPERM; in ep_eventpoll_bp_ioctl()
528 WRITE_ONCE(ep->busy_poll_usecs, epoll_params.busy_poll_usecs); in ep_eventpoll_bp_ioctl()
529 WRITE_ONCE(ep->busy_poll_budget, epoll_params.busy_poll_budget); in ep_eventpoll_bp_ioctl()
530 WRITE_ONCE(ep->prefer_busy_poll, epoll_params.prefer_busy_poll); in ep_eventpoll_bp_ioctl()
534 epoll_params.busy_poll_usecs = READ_ONCE(ep->busy_poll_usecs); in ep_eventpoll_bp_ioctl()
535 epoll_params.busy_poll_budget = READ_ONCE(ep->busy_poll_budget); in ep_eventpoll_bp_ioctl()
536 epoll_params.prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll); in ep_eventpoll_bp_ioctl()
538 return -EFAULT; in ep_eventpoll_bp_ioctl()
541 return -ENOIOCTLCMD; in ep_eventpoll_bp_ioctl()
547 unsigned int napi_id = READ_ONCE(ep->napi_id); in ep_suspend_napi_irqs()
549 if (napi_id >= MIN_NAPI_ID && READ_ONCE(ep->prefer_busy_poll)) in ep_suspend_napi_irqs()
555 unsigned int napi_id = READ_ONCE(ep->napi_id); in ep_resume_napi_irqs()
557 if (napi_id >= MIN_NAPI_ID && READ_ONCE(ep->prefer_busy_poll)) in ep_resume_napi_irqs()
575 return -EOPNOTSUPP; in ep_eventpoll_bp_ioctl()
590 * the use of wait queues used by epoll is done in a very controlled
605 * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()
624 * it might be natural to create a per-cpu nest count. However, since in ep_poll_safewake()
625 * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can in ep_poll_safewake()
626 * schedule() in the -rt kernel, the per-cpu variable are no longer in ep_poll_safewake()
638 if ((is_file_epoll(epi->ffd.file))) { in ep_poll_safewake()
639 ep_src = epi->ffd.file->private_data; in ep_poll_safewake()
640 nests = ep_src->nests; in ep_poll_safewake()
645 spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests); in ep_poll_safewake()
646 ep->nests = nests + 1; in ep_poll_safewake()
647 wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags); in ep_poll_safewake()
648 ep->nests = 0; in ep_poll_safewake()
649 spin_unlock_irqrestore(&ep->poll_wait.lock, flags); in ep_poll_safewake()
657 wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags); in ep_poll_safewake()
668 * If it is cleared by POLLFREE, it should be rcu-safe. in ep_remove_wait_queue()
671 * we rely on whead->lock. in ep_remove_wait_queue()
673 whead = smp_load_acquire(&pwq->whead); in ep_remove_wait_queue()
675 remove_wait_queue(whead, &pwq->wait); in ep_remove_wait_queue()
685 struct eppoll_entry **p = &epi->pwqlist; in ep_unregister_pollwait()
689 *p = pwq->next; in ep_unregister_pollwait()
695 /* call only when ep->mtx is held */
698 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
701 /* call only when ep->mtx is held */
712 return rcu_access_pointer(epi->ws) ? true : false; in ep_has_wakeup_source()
715 /* call when ep->mtx cannot be held (ep_poll_callback) */
721 ws = rcu_dereference(epi->ws); in ep_pm_stay_awake_rcu()
729 * ep->mutex needs to be held because we could be hit by
735 * Steal the ready list, and re-init the original one to the in ep_start_scan()
736 * empty list. Also, set ep->ovflist to NULL so that events in ep_start_scan()
738 * have the poll callback to queue directly on ep->rdllist, in ep_start_scan()
743 write_lock_irq(&ep->lock); in ep_start_scan()
744 list_splice_init(&ep->rdllist, txlist); in ep_start_scan()
745 WRITE_ONCE(ep->ovflist, NULL); in ep_start_scan()
746 write_unlock_irq(&ep->lock); in ep_start_scan()
754 write_lock_irq(&ep->lock); in ep_done_scan()
758 * We re-insert them inside the main ready-list here. in ep_done_scan()
760 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; in ep_done_scan()
761 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { in ep_done_scan()
765 * queued into ->ovflist but the "txlist" might already in ep_done_scan()
770 * ->ovflist is LIFO, so we have to reverse it in order in ep_done_scan()
773 list_add(&epi->rdllink, &ep->rdllist); in ep_done_scan()
778 * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after in ep_done_scan()
780 * ep->rdllist. in ep_done_scan()
782 WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); in ep_done_scan()
785 * Quickly re-inject items left on "txlist". in ep_done_scan()
787 list_splice(txlist, &ep->rdllist); in ep_done_scan()
788 __pm_relax(ep->ws); in ep_done_scan()
790 if (!list_empty(&ep->rdllist)) { in ep_done_scan()
791 if (waitqueue_active(&ep->wq)) in ep_done_scan()
792 wake_up(&ep->wq); in ep_done_scan()
795 write_unlock_irq(&ep->lock); in ep_done_scan()
800 refcount_inc(&ep->refcount); in ep_get()
808 if (!refcount_dec_and_test(&ep->refcount)) in ep_refcount_dec_and_test()
811 WARN_ON_ONCE(!RB_EMPTY_ROOT(&ep->rbr.rb_root)); in ep_refcount_dec_and_test()
818 mutex_destroy(&ep->mtx); in ep_free()
819 free_uid(ep->user); in ep_free()
820 wakeup_source_unregister(ep->ws); in ep_free()
834 struct file *file = epi->ffd.file; in __ep_remove()
841 * Removes poll wait queue hooks. in __ep_remove()
846 spin_lock(&file->f_lock); in __ep_remove()
847 if (epi->dying && !force) { in __ep_remove()
848 spin_unlock(&file->f_lock); in __ep_remove()
853 head = file->f_ep; in __ep_remove()
854 if (head->first == &epi->fllink && !epi->fllink.next) { in __ep_remove()
856 WRITE_ONCE(file->f_ep, NULL); in __ep_remove()
860 if (!smp_load_acquire(&v->next)) in __ep_remove()
864 hlist_del_rcu(&epi->fllink); in __ep_remove()
865 spin_unlock(&file->f_lock); in __ep_remove()
868 rb_erase_cached(&epi->rbn, &ep->rbr); in __ep_remove()
870 write_lock_irq(&ep->lock); in __ep_remove()
872 list_del_init(&epi->rdllink); in __ep_remove()
873 write_unlock_irq(&ep->lock); in __ep_remove()
877 * At this point it is safe to free the eventpoll item. Use the union in __ep_remove()
878 * field epi->rcu, since we are trying to minimize the size of in __ep_remove()
880 * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make in __ep_remove()
885 percpu_counter_dec(&ep->user->epoll_watches); in __ep_remove()
904 if (waitqueue_active(&ep->poll_wait)) in ep_clear_and_put()
907 mutex_lock(&ep->mtx); in ep_clear_and_put()
912 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_clear_and_put()
920 * Walks through the whole tree and try to free each "struct epitem". in ep_clear_and_put()
927 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) { in ep_clear_and_put()
935 mutex_unlock(&ep->mtx); in ep_clear_and_put()
947 return -EINVAL; in ep_eventpoll_ioctl()
955 ret = -EINVAL; in ep_eventpoll_ioctl()
964 struct eventpoll *ep = file->private_data; in ep_eventpoll_release()
974 static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth) in __ep_eventpoll_poll() argument
976 struct eventpoll *ep = file->private_data; in __ep_eventpoll_poll()
984 /* Insert inside our poll wait queue */ in __ep_eventpoll_poll()
985 poll_wait(file, &ep->poll_wait, wait); in __ep_eventpoll_poll()
991 mutex_lock_nested(&ep->mtx, depth); in __ep_eventpoll_poll()
1004 list_del_init(&epi->rdllink); in __ep_eventpoll_poll()
1008 mutex_unlock(&ep->mtx); in __ep_eventpoll_poll()
1017 * been free'd and then gotten re-allocated to something else (since
1018 * files are not RCU-delayed, they are SLAB_TYPESAFE_BY_RCU).
1020 * But for epoll, users hold the ep->mtx mutex, and as such any file in
1021 * the process of being free'd will block in eventpoll_release_file()
1022 * and thus the underlying file allocation will not be free'd, and the
1023 * file re-use cannot happen.
1026 * operation - 'ffd.file' cannot go away even if the refcount has
1027 * reached zero (but we must still not call out to ->poll() functions
1034 file = epi->ffd.file; in epi_fget()
1035 if (!file_ref_get(&file->f_ref)) in epi_fget()
1042 * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
1058 pt->_key = epi->event.events; in ep_item_poll()
1064 return res & epi->event.events; in ep_item_poll()
1067 static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait) in ep_eventpoll_poll() argument
1069 return __ep_eventpoll_poll(file, wait, 0); in ep_eventpoll_poll()
1075 struct eventpoll *ep = f->private_data; in ep_show_fdinfo()
1078 mutex_lock(&ep->mtx); in ep_show_fdinfo()
1079 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_show_fdinfo()
1081 struct inode *inode = file_inode(epi->ffd.file); in ep_show_fdinfo()
1085 epi->ffd.fd, epi->event.events, in ep_show_fdinfo()
1086 (long long)epi->event.data, in ep_show_fdinfo()
1087 (long long)epi->ffd.file->f_pos, in ep_show_fdinfo()
1088 inode->i_ino, inode->i_sb->s_dev); in ep_show_fdinfo()
1092 mutex_unlock(&ep->mtx); in ep_show_fdinfo()
1122 * the ep->mtx. in eventpoll_release_file()
1125 spin_lock(&file->f_lock); in eventpoll_release_file()
1126 if (file->f_ep && file->f_ep->first) { in eventpoll_release_file()
1127 epi = hlist_entry(file->f_ep->first, struct epitem, fllink); in eventpoll_release_file()
1128 epi->dying = true; in eventpoll_release_file()
1129 spin_unlock(&file->f_lock); in eventpoll_release_file()
1135 ep = epi->ep; in eventpoll_release_file()
1136 mutex_lock(&ep->mtx); in eventpoll_release_file()
1138 mutex_unlock(&ep->mtx); in eventpoll_release_file()
1144 spin_unlock(&file->f_lock); in eventpoll_release_file()
1153 return -ENOMEM; in ep_alloc()
1155 mutex_init(&ep->mtx); in ep_alloc()
1156 rwlock_init(&ep->lock); in ep_alloc()
1157 init_waitqueue_head(&ep->wq); in ep_alloc()
1158 init_waitqueue_head(&ep->poll_wait); in ep_alloc()
1159 INIT_LIST_HEAD(&ep->rdllist); in ep_alloc()
1160 ep->rbr = RB_ROOT_CACHED; in ep_alloc()
1161 ep->ovflist = EP_UNACTIVE_PTR; in ep_alloc()
1162 ep->user = get_current_user(); in ep_alloc()
1163 refcount_set(&ep->refcount, 1); in ep_alloc()
1183 for (rbp = ep->rbr.rb_root.rb_node; rbp; ) { in ep_find()
1185 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); in ep_find()
1187 rbp = rbp->rb_right; in ep_find()
1189 rbp = rbp->rb_left; in ep_find()
1205 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_find_tfd()
1207 if (epi->ffd.fd == tfd) { in ep_find_tfd()
1211 toff--; in ep_find_tfd()
1227 return ERR_PTR(-EINVAL); in get_epoll_tfile_raw_ptr()
1229 ep = file->private_data; in get_epoll_tfile_raw_ptr()
1231 mutex_lock(&ep->mtx); in get_epoll_tfile_raw_ptr()
1234 file_raw = epi->ffd.file; in get_epoll_tfile_raw_ptr()
1236 file_raw = ERR_PTR(-ENOENT); in get_epoll_tfile_raw_ptr()
1237 mutex_unlock(&ep->mtx); in get_epoll_tfile_raw_ptr()
1267 * This is simple 'new->next = head' operation, but cmpxchg() in list_add_tail_lockless()
1270 * new->next == new. in list_add_tail_lockless()
1272 if (!try_cmpxchg(&new->next, &new, head)) in list_add_tail_lockless()
1276 * Initially ->next of a new element must be updated with the head in list_add_tail_lockless()
1278 * exchanged. XCHG guarantees memory ordering, thus ->next should be in list_add_tail_lockless()
1280 * swapped before prev->next is updated. in list_add_tail_lockless()
1283 prev = xchg(&head->prev, new); in list_add_tail_lockless()
1286 * It is safe to modify prev->next and new->prev, because a new element in list_add_tail_lockless()
1287 * is added only to the tail and new->next is updated before XCHG. in list_add_tail_lockless()
1290 prev->next = new; in list_add_tail_lockless()
1291 new->prev = prev; in list_add_tail_lockless()
1297 * Chains a new epi entry to the tail of the ep->ovflist in a lockless way,
1304 struct eventpoll *ep = epi->ep; in chain_epi_lockless()
1307 if (epi->next != EP_UNACTIVE_PTR) in chain_epi_lockless()
1311 if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR) in chain_epi_lockless()
1315 epi->next = xchg(&ep->ovflist, epi); in chain_epi_lockless()
1321 * This is the callback that is passed to the wait queue wakeup
1326 * events from another file descriptor, thus all modifications to ->rdllist
1327 * or ->ovflist are lockless. Read lock is paired with the write lock from
1333 * with several wait queues entries. Plural wakeup from different CPUs of a
1334 * single wait queue is serialized by wq.lock, but the case when multiple wait
1338 static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) in ep_poll_callback() argument
1341 struct epitem *epi = ep_item_from_wait(wait); in ep_poll_callback()
1342 struct eventpoll *ep = epi->ep; in ep_poll_callback()
1347 read_lock_irqsave(&ep->lock, flags); in ep_poll_callback()
1357 if (!(epi->event.events & ~EP_PRIVATE_BITS)) in ep_poll_callback()
1366 if (pollflags && !(pollflags & epi->event.events)) in ep_poll_callback()
1371 * (because we're accessing user memory, and because of linux f_op->poll() in ep_poll_callback()
1373 * chained in ep->ovflist and requeued later on. in ep_poll_callback()
1375 if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) { in ep_poll_callback()
1380 if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist)) in ep_poll_callback()
1385 * Wake up ( if active ) both the eventpoll wait list and the ->poll() in ep_poll_callback()
1386 * wait list. in ep_poll_callback()
1388 if (waitqueue_active(&ep->wq)) { in ep_poll_callback()
1389 if ((epi->event.events & EPOLLEXCLUSIVE) && in ep_poll_callback()
1393 if (epi->event.events & EPOLLIN) in ep_poll_callback()
1397 if (epi->event.events & EPOLLOUT) in ep_poll_callback()
1406 wake_up_sync(&ep->wq); in ep_poll_callback()
1408 wake_up(&ep->wq); in ep_poll_callback()
1410 if (waitqueue_active(&ep->poll_wait)) in ep_poll_callback()
1414 read_unlock_irqrestore(&ep->lock, flags); in ep_poll_callback()
1420 if (!(epi->event.events & EPOLLEXCLUSIVE)) in ep_poll_callback()
1426 * ->whead = NULL and do another remove_wait_queue() after in ep_poll_callback()
1427 * us, so we can't use __remove_wait_queue(). in ep_poll_callback()
1429 list_del_init(&wait->entry); in ep_poll_callback()
1431 * ->whead != NULL protects us from the race with in ep_poll_callback()
1433 * takes whead->lock held by the caller. Once we nullify it, in ep_poll_callback()
1434 * nothing protects ep/epi or even wait. in ep_poll_callback()
1436 smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL); in ep_poll_callback()
1443 * This is the callback that is used to add our wait queue to the
1450 struct epitem *epi = epq->epi; in ep_ptable_queue_proc()
1458 epq->epi = NULL; in ep_ptable_queue_proc()
1462 init_waitqueue_func_entry(&pwq->wait, ep_poll_callback); in ep_ptable_queue_proc()
1463 pwq->whead = whead; in ep_ptable_queue_proc()
1464 pwq->base = epi; in ep_ptable_queue_proc()
1465 if (epi->event.events & EPOLLEXCLUSIVE) in ep_ptable_queue_proc()
1466 add_wait_queue_exclusive(whead, &pwq->wait); in ep_ptable_queue_proc()
1468 add_wait_queue(whead, &pwq->wait); in ep_ptable_queue_proc()
1469 pwq->next = epi->pwqlist; in ep_ptable_queue_proc()
1470 epi->pwqlist = pwq; in ep_ptable_queue_proc()
1476 struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL; in ep_rbtree_insert()
1483 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); in ep_rbtree_insert()
1485 p = &parent->rb_right; in ep_rbtree_insert()
1488 p = &parent->rb_left; in ep_rbtree_insert()
1490 rb_link_node(&epi->rbn, parent, p); in ep_rbtree_insert()
1491 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost); in ep_rbtree_insert()
1518 return -1; in path_count_inc()
1536 return -1; in reverse_path_check_proc()
1540 struct hlist_head *refs = &epi->ep->refs; in reverse_path_check_proc()
1552 * reverse_path_check - The tfile_check_list is list of epitem_head, which have
1559 * %-1 otherwise.
1565 for (p = tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) { in reverse_path_check()
1569 error = reverse_path_check_proc(&p->epitems, 0); in reverse_path_check()
1582 if (!epi->ep->ws) { in ep_create_wakeup_source()
1583 epi->ep->ws = wakeup_source_register(NULL, "eventpoll"); in ep_create_wakeup_source()
1584 if (!epi->ep->ws) in ep_create_wakeup_source()
1585 return -ENOMEM; in ep_create_wakeup_source()
1588 take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry); in ep_create_wakeup_source()
1593 return -ENOMEM; in ep_create_wakeup_source()
1594 rcu_assign_pointer(epi->ws, ws); in ep_create_wakeup_source()
1604 RCU_INIT_POINTER(epi->ws, NULL); in ep_destroy_wakeup_source()
1607 * wait for ep_pm_stay_awake_rcu to finish, synchronize_rcu is in ep_destroy_wakeup_source()
1622 ep = file->private_data; in attach_epitem()
1625 head = &ep->refs; in attach_epitem()
1626 } else if (!READ_ONCE(file->f_ep)) { in attach_epitem()
1630 return -ENOMEM; in attach_epitem()
1631 head = &to_free->epitems; in attach_epitem()
1633 spin_lock(&file->f_lock); in attach_epitem()
1634 if (!file->f_ep) { in attach_epitem()
1636 spin_unlock(&file->f_lock); in attach_epitem()
1640 WRITE_ONCE(file->f_ep, head); in attach_epitem()
1643 hlist_add_head_rcu(&epi->fllink, file->f_ep); in attach_epitem()
1644 spin_unlock(&file->f_lock); in attach_epitem()
1662 tep = tfile->private_data; in ep_insert()
1666 if (unlikely(percpu_counter_compare(&ep->user->epoll_watches, in ep_insert()
1668 return -ENOSPC; in ep_insert()
1669 percpu_counter_inc(&ep->user->epoll_watches); in ep_insert()
1672 percpu_counter_dec(&ep->user->epoll_watches); in ep_insert()
1673 return -ENOMEM; in ep_insert()
1677 INIT_LIST_HEAD(&epi->rdllink); in ep_insert()
1678 epi->ep = ep; in ep_insert()
1679 ep_set_ffd(&epi->ffd, tfile, fd); in ep_insert()
1680 epi->event = *event; in ep_insert()
1681 epi->next = EP_UNACTIVE_PTR; in ep_insert()
1684 mutex_lock_nested(&tep->mtx, 1); in ep_insert()
1688 mutex_unlock(&tep->mtx); in ep_insert()
1690 percpu_counter_dec(&ep->user->epoll_watches); in ep_insert()
1691 return -ENOMEM; in ep_insert()
1703 mutex_unlock(&tep->mtx); in ep_insert()
1714 return -EINVAL; in ep_insert()
1717 if (epi->event.events & EPOLLWAKEUP) { in ep_insert()
1739 * We have to check if something went wrong during the poll wait queue in ep_insert()
1740 * install process. Namely an allocation for a wait queue failed due in ep_insert()
1745 return -ENOMEM; in ep_insert()
1749 write_lock_irq(&ep->lock); in ep_insert()
1756 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1760 if (waitqueue_active(&ep->wq)) in ep_insert()
1761 wake_up(&ep->wq); in ep_insert()
1762 if (waitqueue_active(&ep->poll_wait)) in ep_insert()
1766 write_unlock_irq(&ep->lock); in ep_insert()
1790 * Set the new event interest mask before calling f_op->poll(); in ep_modify()
1792 * f_op->poll() call and the new event set registering. in ep_modify()
1794 epi->event.events = event->events; /* need barrier below */ in ep_modify()
1795 epi->event.data = event->data; /* protected by mtx */ in ep_modify()
1796 if (epi->event.events & EPOLLWAKEUP) { in ep_modify()
1808 * event occurs immediately after we call f_op->poll(). in ep_modify()
1809 * We need this because we did not take ep->lock while in ep_modify()
1811 * ep->lock). in ep_modify()
1814 * when calling f_op->poll(). This barrier also in ep_modify()
1818 * This barrier will now guarantee ep_poll_callback or f_op->poll in ep_modify()
1830 write_lock_irq(&ep->lock); in ep_modify()
1832 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1836 if (waitqueue_active(&ep->wq)) in ep_modify()
1837 wake_up(&ep->wq); in ep_modify()
1838 if (waitqueue_active(&ep->poll_wait)) in ep_modify()
1841 write_unlock_irq(&ep->lock); in ep_modify()
1860 * Always short-circuit for fatal signals to allow threads to make a in ep_send_events()
1865 return -EINTR; in ep_send_events()
1869 mutex_lock(&ep->mtx); in ep_send_events()
1874 * Items cannot vanish during the loop we are holding ep->mtx. in ep_send_events()
1884 * Activate ep->ws before deactivating epi->ws to prevent in ep_send_events()
1885 * triggering auto-suspend here (in case we reactive epi->ws in ep_send_events()
1888 * This could be rearranged to delay the deactivation of epi->ws in ep_send_events()
1889 * instead, but then epi->ws would temporarily be out of sync in ep_send_events()
1894 if (ws->active) in ep_send_events()
1895 __pm_stay_awake(ep->ws); in ep_send_events()
1899 list_del_init(&epi->rdllink); in ep_send_events()
1902 * If the event mask intersect the caller-requested one, in ep_send_events()
1903 * deliver the event to userspace. Again, we are holding ep->mtx, in ep_send_events()
1910 events = epoll_put_uevent(revents, epi->event.data, events); in ep_send_events()
1912 list_add(&epi->rdllink, &txlist); in ep_send_events()
1915 res = -EFAULT; in ep_send_events()
1919 if (epi->event.events & EPOLLONESHOT) in ep_send_events()
1920 epi->event.events &= EP_PRIVATE_BITS; in ep_send_events()
1921 else if (!(epi->event.events & EPOLLET)) { in ep_send_events()
1928 * into ep->rdllist besides us. The epoll_ctl() in ep_send_events()
1931 * poll callback will queue them in ep->ovflist. in ep_send_events()
1933 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events()
1938 mutex_unlock(&ep->mtx); in ep_send_events()
1951 to->tv_sec = 0; in ep_timeout_to_timespec()
1952 to->tv_nsec = 0; in ep_timeout_to_timespec()
1956 to->tv_sec = ms / MSEC_PER_SEC; in ep_timeout_to_timespec()
1957 to->tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC); in ep_timeout_to_timespec()
1979 list_del_init_careful(&wq_entry->entry); in ep_autoremove_wake_function()
2008 * ep_poll - Retrieves ready events, and delivers them to the caller-supplied
2029 wait_queue_entry_t wait; in ep_poll() local
2034 if (timeout && (timeout->tv_sec | timeout->tv_nsec)) { in ep_poll()
2040 * Avoid the unnecessary trip to the wait queue loop, if the in ep_poll()
2049 * with a non-zero timeout, this thread will check the ready list under in ep_poll()
2050 * lock and will add to the wait queue. For cases with a zero in ep_poll()
2071 return -EINTR; in ep_poll()
2075 * thus wait entry is removed from the wait queue on each in ep_poll()
2079 * lost. This is also good performance-wise, because on in ep_poll()
2081 * explicitly, thus ep->lock is not taken, which halts the in ep_poll()
2085 * unconditionally removes, because we don't reuse the wait in ep_poll()
2086 * entry between loop iterations. This lets us also avoid the in ep_poll()
2090 init_wait(&wait); in ep_poll()
2091 wait.func = ep_autoremove_wake_function; in ep_poll()
2093 write_lock_irq(&ep->lock); in ep_poll()
2103 * plays with two lists (->rdllist and ->ovflist) and there in ep_poll()
2110 __add_wait_queue_exclusive(&ep->wq, &wait); in ep_poll()
2112 write_unlock_irq(&ep->lock); in ep_poll()
2121 * If timed out and still on the wait queue, recheck eavail in ep_poll()
2126 if (!list_empty_careful(&wait.entry)) { in ep_poll()
2127 write_lock_irq(&ep->lock); in ep_poll()
2129 * If the thread timed out and is not on the wait queue, in ep_poll()
2132 * Thus, when wait.entry is empty, it needs to harvest in ep_poll()
2136 eavail = list_empty(&wait.entry); in ep_poll()
2137 __remove_wait_queue(&ep->wq, &wait); in ep_poll()
2138 write_unlock_irq(&ep->lock); in ep_poll()
2144 * ep_loop_check_proc - verify that adding an epoll file inside another
2153 * structure @ep does not violate the constraints, or %-1 otherwise.
2161 mutex_lock_nested(&ep->mtx, depth + 1); in ep_loop_check_proc()
2162 ep->gen = loop_check_gen; in ep_loop_check_proc()
2163 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_loop_check_proc()
2165 if (unlikely(is_file_epoll(epi->ffd.file))) { in ep_loop_check_proc()
2167 ep_tovisit = epi->ffd.file->private_data; in ep_loop_check_proc()
2168 if (ep_tovisit->gen == loop_check_gen) in ep_loop_check_proc()
2171 error = -1; in ep_loop_check_proc()
2185 list_file(epi->ffd.file); in ep_loop_check_proc()
2188 mutex_unlock(&ep->mtx); in ep_loop_check_proc()
2194 * ep_loop_check - Performs a check to verify that adding an epoll file (@to)
2202 * does not violate the constraints, or %-1 otherwise.
2215 tfile_check_list = head->next; in clear_tfile_check_list()
2234 return -EINVAL; in do_epoll_create()
2243 * a file structure and a free file descriptor. in do_epoll_create()
2256 ep->file = file; in do_epoll_create()
2275 return -EINVAL; in SYSCALL_DEFINE1()
2283 if ((epev->events & EPOLLWAKEUP) && !capable(CAP_BLOCK_SUSPEND)) in ep_take_care_of_epollwakeup()
2284 epev->events &= ~EPOLLWAKEUP; in ep_take_care_of_epollwakeup()
2289 epev->events &= ~EPOLLWAKEUP; in ep_take_care_of_epollwakeup()
2302 return -EAGAIN; in epoll_mutex_lock()
2316 return -EBADF; in do_epoll_ctl()
2321 return -EBADF; in do_epoll_ctl()
2325 return -EPERM; in do_epoll_ctl()
2333 * the user passed to us _is_ an eventpoll file. And also we do not permit in do_epoll_ctl()
2336 error = -EINVAL; in do_epoll_ctl()
2345 if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) { in do_epoll_ctl()
2349 (epds->events & ~EPOLLEXCLUSIVE_OK_BITS))) in do_epoll_ctl()
2357 ep = fd_file(f)->private_data; in do_epoll_ctl()
2374 error = epoll_mutex_lock(&ep->mtx, 0, nonblock); in do_epoll_ctl()
2378 if (READ_ONCE(fd_file(f)->f_ep) || ep->gen == loop_check_gen || in do_epoll_ctl()
2380 mutex_unlock(&ep->mtx); in do_epoll_ctl()
2387 tep = fd_file(tf)->private_data; in do_epoll_ctl()
2388 error = -ELOOP; in do_epoll_ctl()
2392 error = epoll_mutex_lock(&ep->mtx, 0, nonblock); in do_epoll_ctl()
2405 error = -EINVAL; in do_epoll_ctl()
2409 epds->events |= EPOLLERR | EPOLLHUP; in do_epoll_ctl()
2412 error = -EEXIST; in do_epoll_ctl()
2423 error = -ENOENT; in do_epoll_ctl()
2428 if (!(epi->event.events & EPOLLEXCLUSIVE)) { in do_epoll_ctl()
2429 epds->events |= EPOLLERR | EPOLLHUP; in do_epoll_ctl()
2433 error = -ENOENT; in do_epoll_ctl()
2436 mutex_unlock(&ep->mtx); in do_epoll_ctl()
2459 return -EFAULT; in SYSCALL_DEFINE4()
2465 * Implement the event wait interface for the eventpoll file. It is the kernel
2475 return -EINVAL; in do_epoll_wait()
2479 return -EFAULT; in do_epoll_wait()
2484 return -EBADF; in do_epoll_wait()
2488 * the user passed to us _is_ an eventpoll file. in do_epoll_wait()
2491 return -EINVAL; in do_epoll_wait()
2497 ep = fd_file(f)->private_data; in do_epoll_wait()
2513 * Implement the event wait interface for the eventpoll file. It is the kernel
2523 * If the caller wants a certain signal mask to be set during the wait, in do_epoll_pwait()
2532 restore_saved_sigmask_unless(error == -EINTR); in do_epoll_pwait()
2556 return -EFAULT; in SYSCALL_DEFINE6()
2559 return -EINVAL; in SYSCALL_DEFINE6()
2575 * If the caller wants a certain signal mask to be set during the wait, in do_compat_epoll_pwait()
2584 restore_saved_sigmask_unless(err == -EINTR); in do_compat_epoll_pwait()
2613 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
2616 return -EINVAL; in COMPAT_SYSCALL_DEFINE6()
2633 max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) / in eventpoll_init()
2639 * using an extra cache line on 64-bit (and smaller) CPUs in eventpoll_init()