Lines Matching full:pool
43 #define alloc_stat_inc(pool, __stat) (pool->alloc_stats.__stat++) argument
45 #define recycle_stat_inc(pool, __stat) \ argument
47 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
51 #define recycle_stat_add(pool, __stat, val) \ argument
53 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
72 * page_pool_get_stats() - fetch page pool stats
73 * @pool: pool from which page was allocated
82 bool page_pool_get_stats(const struct page_pool *pool, in page_pool_get_stats() argument
91 stats->alloc_stats.fast += pool->alloc_stats.fast; in page_pool_get_stats()
92 stats->alloc_stats.slow += pool->alloc_stats.slow; in page_pool_get_stats()
93 stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order; in page_pool_get_stats()
94 stats->alloc_stats.empty += pool->alloc_stats.empty; in page_pool_get_stats()
95 stats->alloc_stats.refill += pool->alloc_stats.refill; in page_pool_get_stats()
96 stats->alloc_stats.waive += pool->alloc_stats.waive; in page_pool_get_stats()
100 per_cpu_ptr(pool->recycle_stats, cpu); in page_pool_get_stats()
153 #define alloc_stat_inc(pool, __stat) argument
154 #define recycle_stat_inc(pool, __stat) argument
155 #define recycle_stat_add(pool, __stat, val) argument
158 static bool page_pool_producer_lock(struct page_pool *pool) in page_pool_producer_lock() argument
159 __acquires(&pool->ring.producer_lock) in page_pool_producer_lock()
164 spin_lock(&pool->ring.producer_lock); in page_pool_producer_lock()
166 spin_lock_bh(&pool->ring.producer_lock); in page_pool_producer_lock()
171 static void page_pool_producer_unlock(struct page_pool *pool, in page_pool_producer_unlock() argument
173 __releases(&pool->ring.producer_lock) in page_pool_producer_unlock()
176 spin_unlock(&pool->ring.producer_lock); in page_pool_producer_unlock()
178 spin_unlock_bh(&pool->ring.producer_lock); in page_pool_producer_unlock()
190 static int page_pool_init(struct page_pool *pool, in page_pool_init() argument
200 memcpy(&pool->p, ¶ms->fast, sizeof(pool->p)); in page_pool_init()
201 memcpy(&pool->slow, ¶ms->slow, sizeof(pool->slow)); in page_pool_init()
203 pool->cpuid = cpuid; in page_pool_init()
204 pool->dma_sync_for_cpu = true; in page_pool_init()
207 if (pool->slow.flags & ~PP_FLAG_ALL) in page_pool_init()
210 if (pool->p.pool_size) in page_pool_init()
211 ring_qsize = pool->p.pool_size; in page_pool_init()
221 if (pool->slow.flags & PP_FLAG_DMA_MAP) { in page_pool_init()
222 if ((pool->p.dma_dir != DMA_FROM_DEVICE) && in page_pool_init()
223 (pool->p.dma_dir != DMA_BIDIRECTIONAL)) in page_pool_init()
226 pool->dma_map = true; in page_pool_init()
229 if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) { in page_pool_init()
233 if (!(pool->slow.flags & PP_FLAG_DMA_MAP)) in page_pool_init()
236 if (!pool->p.max_len) in page_pool_init()
239 pool->dma_sync = true; in page_pool_init()
241 /* pool->p.offset has to be set according to the address in page_pool_init()
246 pool->has_init_callback = !!pool->slow.init_callback; in page_pool_init()
249 if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) { in page_pool_init()
250 pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats); in page_pool_init()
251 if (!pool->recycle_stats) in page_pool_init()
254 /* For system page pool instance we use a singular stats object in page_pool_init()
256 * (also percpu) page pool instance. in page_pool_init()
258 pool->recycle_stats = &pp_system_recycle_stats; in page_pool_init()
259 pool->system = true; in page_pool_init()
263 if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) { in page_pool_init()
265 if (!pool->system) in page_pool_init()
266 free_percpu(pool->recycle_stats); in page_pool_init()
271 atomic_set(&pool->pages_state_release_cnt, 0); in page_pool_init()
274 refcount_set(&pool->user_cnt, 1); in page_pool_init()
276 if (pool->dma_map) in page_pool_init()
277 get_device(pool->p.dev); in page_pool_init()
279 if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) { in page_pool_init()
285 rxq = __netif_get_rx_queue(pool->slow.netdev, in page_pool_init()
286 pool->slow.queue_idx); in page_pool_init()
287 pool->mp_priv = rxq->mp_params.mp_priv; in page_pool_init()
290 if (pool->mp_priv) { in page_pool_init()
291 if (!pool->dma_map || !pool->dma_sync) in page_pool_init()
294 err = mp_dmabuf_devmem_init(pool); in page_pool_init()
307 ptr_ring_cleanup(&pool->ring, NULL); in page_pool_init()
309 if (!pool->system) in page_pool_init()
310 free_percpu(pool->recycle_stats); in page_pool_init()
315 static void page_pool_uninit(struct page_pool *pool) in page_pool_uninit() argument
317 ptr_ring_cleanup(&pool->ring, NULL); in page_pool_uninit()
319 if (pool->dma_map) in page_pool_uninit()
320 put_device(pool->p.dev); in page_pool_uninit()
323 if (!pool->system) in page_pool_uninit()
324 free_percpu(pool->recycle_stats); in page_pool_uninit()
329 * page_pool_create_percpu() - create a page pool for a given cpu.
336 struct page_pool *pool; in page_pool_create_percpu() local
339 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid); in page_pool_create_percpu()
340 if (!pool) in page_pool_create_percpu()
343 err = page_pool_init(pool, params, cpuid); in page_pool_create_percpu()
347 err = page_pool_list(pool); in page_pool_create_percpu()
351 return pool; in page_pool_create_percpu()
354 page_pool_uninit(pool); in page_pool_create_percpu()
357 kfree(pool); in page_pool_create_percpu()
363 * page_pool_create() - create a page pool
372 static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem);
374 static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool) in page_pool_refill_alloc_cache() argument
376 struct ptr_ring *r = &pool->ring; in page_pool_refill_alloc_cache()
382 alloc_stat_inc(pool, empty); in page_pool_refill_alloc_cache()
390 pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid; in page_pool_refill_alloc_cache()
392 /* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */ in page_pool_refill_alloc_cache()
403 pool->alloc.cache[pool->alloc.count++] = netmem; in page_pool_refill_alloc_cache()
410 page_pool_return_page(pool, netmem); in page_pool_refill_alloc_cache()
411 alloc_stat_inc(pool, waive); in page_pool_refill_alloc_cache()
415 } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); in page_pool_refill_alloc_cache()
418 if (likely(pool->alloc.count > 0)) { in page_pool_refill_alloc_cache()
419 netmem = pool->alloc.cache[--pool->alloc.count]; in page_pool_refill_alloc_cache()
420 alloc_stat_inc(pool, refill); in page_pool_refill_alloc_cache()
427 static netmem_ref __page_pool_get_cached(struct page_pool *pool) in __page_pool_get_cached() argument
432 if (likely(pool->alloc.count)) { in __page_pool_get_cached()
434 netmem = pool->alloc.cache[--pool->alloc.count]; in __page_pool_get_cached()
435 alloc_stat_inc(pool, fast); in __page_pool_get_cached()
437 netmem = page_pool_refill_alloc_cache(pool); in __page_pool_get_cached()
443 static void __page_pool_dma_sync_for_device(const struct page_pool *pool, in __page_pool_dma_sync_for_device() argument
450 dma_sync_size = min(dma_sync_size, pool->p.max_len); in __page_pool_dma_sync_for_device()
451 __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset, in __page_pool_dma_sync_for_device()
452 dma_sync_size, pool->p.dma_dir); in __page_pool_dma_sync_for_device()
457 page_pool_dma_sync_for_device(const struct page_pool *pool, in page_pool_dma_sync_for_device() argument
461 if (pool->dma_sync && dma_dev_need_sync(pool->p.dev)) in page_pool_dma_sync_for_device()
462 __page_pool_dma_sync_for_device(pool, netmem, dma_sync_size); in page_pool_dma_sync_for_device()
465 static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem) in page_pool_dma_map() argument
472 * This mapping is kept for lifetime of page, until leaving pool. in page_pool_dma_map()
474 dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0, in page_pool_dma_map()
475 (PAGE_SIZE << pool->p.order), pool->p.dma_dir, in page_pool_dma_map()
478 if (dma_mapping_error(pool->p.dev, dma)) in page_pool_dma_map()
484 page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len); in page_pool_dma_map()
490 dma_unmap_page_attrs(pool->p.dev, dma, in page_pool_dma_map()
491 PAGE_SIZE << pool->p.order, pool->p.dma_dir, in page_pool_dma_map()
496 static struct page *__page_pool_alloc_page_order(struct page_pool *pool, in __page_pool_alloc_page_order() argument
502 page = alloc_pages_node(pool->p.nid, gfp, pool->p.order); in __page_pool_alloc_page_order()
506 if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) { in __page_pool_alloc_page_order()
511 alloc_stat_inc(pool, slow_high_order); in __page_pool_alloc_page_order()
512 page_pool_set_pp_info(pool, page_to_netmem(page)); in __page_pool_alloc_page_order()
515 pool->pages_state_hold_cnt++; in __page_pool_alloc_page_order()
516 trace_page_pool_state_hold(pool, page_to_netmem(page), in __page_pool_alloc_page_order()
517 pool->pages_state_hold_cnt); in __page_pool_alloc_page_order()
522 static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool, in __page_pool_alloc_pages_slow() argument
526 unsigned int pp_order = pool->p.order; in __page_pool_alloc_pages_slow()
527 bool dma_map = pool->dma_map; in __page_pool_alloc_pages_slow()
533 return page_to_netmem(__page_pool_alloc_page_order(pool, gfp)); in __page_pool_alloc_pages_slow()
536 if (unlikely(pool->alloc.count > 0)) in __page_pool_alloc_pages_slow()
537 return pool->alloc.cache[--pool->alloc.count]; in __page_pool_alloc_pages_slow()
540 memset(&pool->alloc.cache, 0, sizeof(void *) * bulk); in __page_pool_alloc_pages_slow()
542 nr_pages = alloc_pages_bulk_node(gfp, pool->p.nid, bulk, in __page_pool_alloc_pages_slow()
543 (struct page **)pool->alloc.cache); in __page_pool_alloc_pages_slow()
551 netmem = pool->alloc.cache[i]; in __page_pool_alloc_pages_slow()
552 if (dma_map && unlikely(!page_pool_dma_map(pool, netmem))) { in __page_pool_alloc_pages_slow()
557 page_pool_set_pp_info(pool, netmem); in __page_pool_alloc_pages_slow()
558 pool->alloc.cache[pool->alloc.count++] = netmem; in __page_pool_alloc_pages_slow()
560 pool->pages_state_hold_cnt++; in __page_pool_alloc_pages_slow()
561 trace_page_pool_state_hold(pool, netmem, in __page_pool_alloc_pages_slow()
562 pool->pages_state_hold_cnt); in __page_pool_alloc_pages_slow()
566 if (likely(pool->alloc.count > 0)) { in __page_pool_alloc_pages_slow()
567 netmem = pool->alloc.cache[--pool->alloc.count]; in __page_pool_alloc_pages_slow()
568 alloc_stat_inc(pool, slow); in __page_pool_alloc_pages_slow()
580 netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp) in page_pool_alloc_netmems() argument
585 netmem = __page_pool_get_cached(pool); in page_pool_alloc_netmems()
590 if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_priv) in page_pool_alloc_netmems()
591 netmem = mp_dmabuf_devmem_alloc_netmems(pool, gfp); in page_pool_alloc_netmems()
593 netmem = __page_pool_alloc_pages_slow(pool, gfp); in page_pool_alloc_netmems()
599 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) in page_pool_alloc_pages() argument
601 return netmem_to_page(page_pool_alloc_netmems(pool, gfp)); in page_pool_alloc_pages()
610 s32 page_pool_inflight(const struct page_pool *pool, bool strict) in page_pool_inflight() argument
612 u32 release_cnt = atomic_read(&pool->pages_state_release_cnt); in page_pool_inflight()
613 u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt); in page_pool_inflight()
619 trace_page_pool_release(pool, inflight, hold_cnt, release_cnt); in page_pool_inflight()
629 void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) in page_pool_set_pp_info() argument
631 netmem_set_pp(netmem, pool); in page_pool_set_pp_info()
641 if (pool->has_init_callback) in page_pool_set_pp_info()
642 pool->slow.init_callback(netmem, pool->slow.init_arg); in page_pool_set_pp_info()
651 static __always_inline void __page_pool_release_page_dma(struct page_pool *pool, in __page_pool_release_page_dma() argument
656 if (!pool->dma_map) in __page_pool_release_page_dma()
664 /* When page is unmapped, it cannot be returned to our pool */ in __page_pool_release_page_dma()
665 dma_unmap_page_attrs(pool->p.dev, dma, in __page_pool_release_page_dma()
666 PAGE_SIZE << pool->p.order, pool->p.dma_dir, in __page_pool_release_page_dma()
676 void page_pool_return_page(struct page_pool *pool, netmem_ref netmem) in page_pool_return_page() argument
682 if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_priv) in page_pool_return_page()
683 put = mp_dmabuf_devmem_release_page(pool, netmem); in page_pool_return_page()
685 __page_pool_release_page_dma(pool, netmem); in page_pool_return_page()
687 /* This may be the last page returned, releasing the pool, so in page_pool_return_page()
688 * it is not safe to reference pool afterwards. in page_pool_return_page()
690 count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); in page_pool_return_page()
691 trace_page_pool_state_release(pool, netmem, count); in page_pool_return_page()
697 /* An optimization would be to call __free_pages(page, pool->p.order) in page_pool_return_page()
703 static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem) in page_pool_recycle_in_ring() argument
708 ret = ptr_ring_produce(&pool->ring, (__force void *)netmem); in page_pool_recycle_in_ring()
710 ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem); in page_pool_recycle_in_ring()
713 recycle_stat_inc(pool, ring); in page_pool_recycle_in_ring()
726 struct page_pool *pool) in page_pool_recycle_in_cache() argument
728 if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) { in page_pool_recycle_in_cache()
729 recycle_stat_inc(pool, cache_full); in page_pool_recycle_in_cache()
734 pool->alloc.cache[pool->alloc.count++] = netmem; in page_pool_recycle_in_cache()
735 recycle_stat_inc(pool, cached); in page_pool_recycle_in_cache()
747 * If pool->dma_sync is set, we'll try to sync the DMA area for
748 * the configured size min(dma_sync_size, pool->max_len).
753 __page_pool_put_page(struct page_pool *pool, netmem_ref netmem, in __page_pool_put_page() argument
770 page_pool_dma_sync_for_device(pool, netmem, dma_sync_size); in __page_pool_put_page()
772 if (allow_direct && page_pool_recycle_in_cache(netmem, pool)) in __page_pool_put_page()
792 recycle_stat_inc(pool, released_refcnt); in __page_pool_put_page()
793 page_pool_return_page(pool, netmem); in __page_pool_put_page()
798 static bool page_pool_napi_local(const struct page_pool *pool) in page_pool_napi_local() argument
813 if (READ_ONCE(pool->cpuid) == cpuid) in page_pool_napi_local()
816 napi = READ_ONCE(pool->p.napi); in page_pool_napi_local()
821 void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, in page_pool_put_unrefed_netmem() argument
825 allow_direct = page_pool_napi_local(pool); in page_pool_put_unrefed_netmem()
828 __page_pool_put_page(pool, netmem, dma_sync_size, allow_direct); in page_pool_put_unrefed_netmem()
829 if (netmem && !page_pool_recycle_in_ring(pool, netmem)) { in page_pool_put_unrefed_netmem()
831 recycle_stat_inc(pool, ring_full); in page_pool_put_unrefed_netmem()
832 page_pool_return_page(pool, netmem); in page_pool_put_unrefed_netmem()
837 void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, in page_pool_put_unrefed_page() argument
840 page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size, in page_pool_put_unrefed_page()
845 static void page_pool_recycle_ring_bulk(struct page_pool *pool, in page_pool_recycle_ring_bulk() argument
853 in_softirq = page_pool_producer_lock(pool); in page_pool_recycle_ring_bulk()
856 if (__ptr_ring_produce(&pool->ring, (__force void *)bulk[i])) { in page_pool_recycle_ring_bulk()
858 recycle_stat_inc(pool, ring_full); in page_pool_recycle_ring_bulk()
863 page_pool_producer_unlock(pool, in_softirq); in page_pool_recycle_ring_bulk()
864 recycle_stat_add(pool, ring, i); in page_pool_recycle_ring_bulk()
875 page_pool_return_page(pool, bulk[i]); in page_pool_recycle_ring_bulk()
906 struct page_pool *pool = NULL; in page_pool_put_netmem_bulk() local
917 if (unlikely(!pool)) { in page_pool_put_netmem_bulk()
918 pool = netmem_pp; in page_pool_put_netmem_bulk()
919 allow_direct = page_pool_napi_local(pool); in page_pool_put_netmem_bulk()
920 } else if (netmem_pp != pool) { in page_pool_put_netmem_bulk()
929 netmem = __page_pool_put_page(pool, netmem, -1, in page_pool_put_netmem_bulk()
937 page_pool_recycle_ring_bulk(pool, bulk, bulk_len); in page_pool_put_netmem_bulk()
944 static netmem_ref page_pool_drain_frag(struct page_pool *pool, in page_pool_drain_frag() argument
947 long drain_count = BIAS_MAX - pool->frag_users; in page_pool_drain_frag()
954 page_pool_dma_sync_for_device(pool, netmem, -1); in page_pool_drain_frag()
958 page_pool_return_page(pool, netmem); in page_pool_drain_frag()
962 static void page_pool_free_frag(struct page_pool *pool) in page_pool_free_frag() argument
964 long drain_count = BIAS_MAX - pool->frag_users; in page_pool_free_frag()
965 netmem_ref netmem = pool->frag_page; in page_pool_free_frag()
967 pool->frag_page = 0; in page_pool_free_frag()
972 page_pool_return_page(pool, netmem); in page_pool_free_frag()
975 netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, in page_pool_alloc_frag_netmem() argument
979 unsigned int max_size = PAGE_SIZE << pool->p.order; in page_pool_alloc_frag_netmem()
980 netmem_ref netmem = pool->frag_page; in page_pool_alloc_frag_netmem()
986 *offset = pool->frag_offset; in page_pool_alloc_frag_netmem()
989 netmem = page_pool_drain_frag(pool, netmem); in page_pool_alloc_frag_netmem()
991 recycle_stat_inc(pool, cached); in page_pool_alloc_frag_netmem()
992 alloc_stat_inc(pool, fast); in page_pool_alloc_frag_netmem()
998 netmem = page_pool_alloc_netmems(pool, gfp); in page_pool_alloc_frag_netmem()
1000 pool->frag_page = 0; in page_pool_alloc_frag_netmem()
1004 pool->frag_page = netmem; in page_pool_alloc_frag_netmem()
1007 pool->frag_users = 1; in page_pool_alloc_frag_netmem()
1009 pool->frag_offset = size; in page_pool_alloc_frag_netmem()
1014 pool->frag_users++; in page_pool_alloc_frag_netmem()
1015 pool->frag_offset = *offset + size; in page_pool_alloc_frag_netmem()
1020 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, in page_pool_alloc_frag() argument
1023 return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size, in page_pool_alloc_frag()
1028 static void page_pool_empty_ring(struct page_pool *pool) in page_pool_empty_ring() argument
1033 while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) { in page_pool_empty_ring()
1039 page_pool_return_page(pool, netmem); in page_pool_empty_ring()
1043 static void __page_pool_destroy(struct page_pool *pool) in __page_pool_destroy() argument
1045 if (pool->disconnect) in __page_pool_destroy()
1046 pool->disconnect(pool); in __page_pool_destroy()
1048 page_pool_unlist(pool); in __page_pool_destroy()
1049 page_pool_uninit(pool); in __page_pool_destroy()
1051 if (pool->mp_priv) { in __page_pool_destroy()
1052 mp_dmabuf_devmem_destroy(pool); in __page_pool_destroy()
1056 kfree(pool); in __page_pool_destroy()
1059 static void page_pool_empty_alloc_cache_once(struct page_pool *pool) in page_pool_empty_alloc_cache_once() argument
1063 if (pool->destroy_cnt) in page_pool_empty_alloc_cache_once()
1070 while (pool->alloc.count) { in page_pool_empty_alloc_cache_once()
1071 netmem = pool->alloc.cache[--pool->alloc.count]; in page_pool_empty_alloc_cache_once()
1072 page_pool_return_page(pool, netmem); in page_pool_empty_alloc_cache_once()
1076 static void page_pool_scrub(struct page_pool *pool) in page_pool_scrub() argument
1078 page_pool_empty_alloc_cache_once(pool); in page_pool_scrub()
1079 pool->destroy_cnt++; in page_pool_scrub()
1084 page_pool_empty_ring(pool); in page_pool_scrub()
1087 static int page_pool_release(struct page_pool *pool) in page_pool_release() argument
1091 page_pool_scrub(pool); in page_pool_release()
1092 inflight = page_pool_inflight(pool, true); in page_pool_release()
1094 __page_pool_destroy(pool); in page_pool_release()
1102 struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw); in page_pool_release_retry() local
1106 inflight = page_pool_release(pool); in page_pool_release_retry()
1117 netdev = READ_ONCE(pool->slow.netdev); in page_pool_release_retry()
1118 if (time_after_eq(jiffies, pool->defer_warn) && in page_pool_release_retry()
1120 int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ; in page_pool_release_retry()
1122 pr_warn("%s() stalled pool shutdown: id %u, %d inflight %d sec\n", in page_pool_release_retry()
1123 __func__, pool->user.id, inflight, sec); in page_pool_release_retry()
1124 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_release_retry()
1128 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_release_retry()
1131 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), in page_pool_use_xdp_mem() argument
1134 refcount_inc(&pool->user_cnt); in page_pool_use_xdp_mem()
1135 pool->disconnect = disconnect; in page_pool_use_xdp_mem()
1136 pool->xdp_mem_id = mem->id; in page_pool_use_xdp_mem()
1139 void page_pool_disable_direct_recycling(struct page_pool *pool) in page_pool_disable_direct_recycling() argument
1141 /* Disable direct recycling based on pool->cpuid. in page_pool_disable_direct_recycling()
1144 WRITE_ONCE(pool->cpuid, -1); in page_pool_disable_direct_recycling()
1146 if (!pool->p.napi) in page_pool_disable_direct_recycling()
1150 * pool and NAPI are unlinked when NAPI is disabled. in page_pool_disable_direct_recycling()
1152 WARN_ON(!test_bit(NAPI_STATE_SCHED, &pool->p.napi->state)); in page_pool_disable_direct_recycling()
1153 WARN_ON(READ_ONCE(pool->p.napi->list_owner) != -1); in page_pool_disable_direct_recycling()
1156 WRITE_ONCE(pool->p.napi, NULL); in page_pool_disable_direct_recycling()
1161 void page_pool_destroy(struct page_pool *pool) in page_pool_destroy() argument
1163 if (!pool) in page_pool_destroy()
1166 if (!page_pool_put(pool)) in page_pool_destroy()
1169 page_pool_disable_direct_recycling(pool); in page_pool_destroy()
1170 page_pool_free_frag(pool); in page_pool_destroy()
1172 if (!page_pool_release(pool)) in page_pool_destroy()
1175 page_pool_detached(pool); in page_pool_destroy()
1176 pool->defer_start = jiffies; in page_pool_destroy()
1177 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_destroy()
1179 INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry); in page_pool_destroy()
1180 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_destroy()
1185 void page_pool_update_nid(struct page_pool *pool, int new_nid) in page_pool_update_nid() argument
1189 trace_page_pool_update_nid(pool, new_nid); in page_pool_update_nid()
1190 pool->p.nid = new_nid; in page_pool_update_nid()
1192 /* Flush pool alloc cache, as refill will check NUMA node */ in page_pool_update_nid()
1193 while (pool->alloc.count) { in page_pool_update_nid()
1194 netmem = pool->alloc.cache[--pool->alloc.count]; in page_pool_update_nid()
1195 page_pool_return_page(pool, netmem); in page_pool_update_nid()