Lines Matching full:element
24 static void poison_error(mempool_t *pool, void *element, size_t size, in poison_error() argument
32 pr_err("BUG: mempool element poison mismatch\n"); in poison_error()
34 pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : ""); in poison_error()
36 pr_cont("%x ", *(u8 *)(element + i)); in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size) in __check_element() argument
43 u8 *obj = element; in __check_element()
50 poison_error(pool, element, size, i); in __check_element()
57 static void check_element(mempool_t *pool, void *element) in check_element() argument
59 /* Skip checking: KASAN might save its metadata in the element. */ in check_element()
65 __check_element(pool, element, (size_t)pool->pool_data); in check_element()
67 __check_element(pool, element, kmem_cache_size(pool->pool_data)); in check_element()
71 void *addr = kmap_local_page((struct page *)element); in check_element()
78 static void __poison_element(void *element, size_t size) in __poison_element() argument
80 u8 *obj = element; in __poison_element()
86 static void poison_element(mempool_t *pool, void *element) in poison_element() argument
88 /* Skip poisoning: KASAN might save its metadata in the element. */ in poison_element()
94 __poison_element(element, (size_t)pool->pool_data); in poison_element()
96 __poison_element(element, kmem_cache_size(pool->pool_data)); in poison_element()
100 void *addr = kmap_local_page((struct page *)element); in poison_element()
107 static inline void check_element(mempool_t *pool, void *element) in check_element() argument
110 static inline void poison_element(mempool_t *pool, void *element) in poison_element() argument
115 static __always_inline bool kasan_poison_element(mempool_t *pool, void *element) in kasan_poison_element() argument
118 return kasan_mempool_poison_object(element); in kasan_poison_element()
120 return kasan_mempool_poison_pages(element, in kasan_poison_element()
125 static void kasan_unpoison_element(mempool_t *pool, void *element) in kasan_unpoison_element() argument
128 kasan_mempool_unpoison_object(element, (size_t)pool->pool_data); in kasan_unpoison_element()
130 kasan_mempool_unpoison_object(element, in kasan_unpoison_element()
133 kasan_mempool_unpoison_pages(element, in kasan_unpoison_element()
137 static __always_inline void add_element(mempool_t *pool, void *element) in add_element() argument
140 poison_element(pool, element); in add_element()
141 if (kasan_poison_element(pool, element)) in add_element()
142 pool->elements[pool->curr_nr++] = element; in add_element()
147 void *element = pool->elements[--pool->curr_nr]; in remove_element() local
150 kasan_unpoison_element(pool, element); in remove_element()
151 check_element(pool, element); in remove_element()
152 return element; in remove_element()
169 void *element = remove_element(pool); in mempool_exit() local
170 pool->free(element, pool->pool_data); in mempool_exit()
215 void *element; in mempool_init_node() local
217 element = pool->alloc(gfp_mask, pool->pool_data); in mempool_init_node()
218 if (unlikely(!element)) { in mempool_init_node()
222 add_element(pool, element); in mempool_init_node()
234 * @alloc_fn: user-defined element-allocation function.
235 * @free_fn: user-defined element-freeing function.
256 * @alloc_fn: user-defined element-allocation function.
257 * @free_fn: user-defined element-freeing function.
310 void *element; in mempool_resize() local
320 element = remove_element(pool); in mempool_resize()
322 pool->free(element, pool->pool_data); in mempool_resize()
351 element = pool->alloc(GFP_KERNEL, pool->pool_data); in mempool_resize()
352 if (!element) in mempool_resize()
356 add_element(pool, element); in mempool_resize()
359 pool->free(element, pool->pool_data); /* Raced */ in mempool_resize()
371 * mempool_alloc - allocate an element from a specific memory pool
382 * Return: pointer to the allocated element or %NULL on error.
386 void *element; in mempool_alloc_noprof() local
402 element = pool->alloc(gfp_temp, pool->pool_data); in mempool_alloc_noprof()
403 if (likely(element != NULL)) in mempool_alloc_noprof()
404 return element; in mempool_alloc_noprof()
408 element = remove_element(pool); in mempool_alloc_noprof()
416 kmemleak_update_trace(element); in mempool_alloc_noprof()
417 return element; in mempool_alloc_noprof()
436 /* Let's wait for someone else to return an element to @pool */ in mempool_alloc_noprof()
454 * mempool_alloc_preallocated - allocate an element from preallocated elements
460 * an element from the preallocated elements. It does not sleep and immediately
463 * Return: pointer to the allocated element or %NULL if no elements are
468 void *element; in mempool_alloc_preallocated() local
473 element = remove_element(pool); in mempool_alloc_preallocated()
481 kmemleak_update_trace(element); in mempool_alloc_preallocated()
482 return element; in mempool_alloc_preallocated()
491 * mempool_free - return an element to the pool.
492 * @element: pool element pointer.
498 void mempool_free(void *element, mempool_t *pool) in mempool_free() argument
502 if (unlikely(element == NULL)) in mempool_free()
507 * for @element and the following @pool->curr_nr. This ensures in mempool_free()
509 * allocation of @element. This is necessary for fringe cases in mempool_free()
510 * where @element was passed to this task without going through in mempool_free()
528 * allocation of @element, any task which decremented curr_nr below in mempool_free()
531 * to min_nr after the allocation of @element, the elements in mempool_free()
541 add_element(pool, element); in mempool_free()
548 pool->free(element, pool->pool_data); in mempool_free()
563 void mempool_free_slab(void *element, void *pool_data) in mempool_free_slab() argument
566 kmem_cache_free(mem, element); in mempool_free_slab()
581 void mempool_kfree(void *element, void *pool_data) in mempool_kfree() argument
583 kfree(element); in mempool_kfree()
594 void mempool_kvfree(void *element, void *pool_data) in mempool_kvfree() argument
596 kvfree(element); in mempool_kvfree()
611 void mempool_free_pages(void *element, void *pool_data) in mempool_free_pages() argument
614 __free_pages(element, order); in mempool_free_pages()