24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
32 	pr_err("BUG: mempool element poison mismatch\n");  in poison_error()
34 	pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");  in poison_error()
36 		pr_cont("%x ", *(u8 *)(element + i));  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
43 	u8 *obj = element;  in __check_element()
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
59 	/* Skip checking: KASAN might save its metadata in the element. */  in check_element()
65 		__check_element(pool, element, (size_t)pool->pool_data);  in check_element()
67 		__check_element(pool, element, kmem_cache_size(pool->pool_data));  in check_element()
71 		void *addr = kmap_local_page((struct page *)element);  in check_element()
78 static void __poison_element(void *element, size_t size)  in __poison_element()  argument
80 	u8 *obj = element;  in __poison_element()
86 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
88 	/* Skip poisoning: KASAN might save its metadata in the element. */  in poison_element()
94 		__poison_element(element, (size_t)pool->pool_data);  in poison_element()
96 		__poison_element(element, kmem_cache_size(pool->pool_data));  in poison_element()
100 		void *addr = kmap_local_page((struct page *)element);  in poison_element()
107 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
110 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
115 static __always_inline bool kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
118 		return kasan_mempool_poison_object(element);  in kasan_poison_element()
120 		return kasan_mempool_poison_pages(element,  in kasan_poison_element()
125 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
128 		kasan_mempool_unpoison_object(element, (size_t)pool->pool_data);  in kasan_unpoison_element()
130 		kasan_mempool_unpoison_object(element,  in kasan_unpoison_element()
133 		kasan_mempool_unpoison_pages(element,  in kasan_unpoison_element()
137 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
140 	poison_element(pool, element);  in add_element()
141 	if (kasan_poison_element(pool, element))  in add_element()
142 		pool->elements[pool->curr_nr++] = element;  in add_element()
147 	void *element = pool->elements[--pool->curr_nr];  in remove_element()  local
150 	kasan_unpoison_element(pool, element);  in remove_element()
151 	check_element(pool, element);  in remove_element()
152 	return element;  in remove_element()
169 		void *element = remove_element(pool);  in mempool_exit()  local
170 		pool->free(element, pool->pool_data);  in mempool_exit()
215 		void *element;  in mempool_init_node()  local
217 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
218 		if (unlikely(!element)) {  in mempool_init_node()
222 		add_element(pool, element);  in mempool_init_node()
234  * @alloc_fn:  user-defined element-allocation function.
235  * @free_fn:   user-defined element-freeing function.
256  * @alloc_fn:  user-defined element-allocation function.
257  * @free_fn:   user-defined element-freeing function.
310 	void *element;  in mempool_resize()  local
320 			element = remove_element(pool);  in mempool_resize()
322 			pool->free(element, pool->pool_data);  in mempool_resize()
351 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
352 		if (!element)  in mempool_resize()
356 			add_element(pool, element);  in mempool_resize()
359 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
371  * mempool_alloc - allocate an element from a specific memory pool
382  * Return: pointer to the allocated element or %NULL on error.
386 	void *element;  in mempool_alloc_noprof()  local
402 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc_noprof()
403 	if (likely(element != NULL))  in mempool_alloc_noprof()
404 		return element;  in mempool_alloc_noprof()
408 		element = remove_element(pool);  in mempool_alloc_noprof()
416 		kmemleak_update_trace(element);  in mempool_alloc_noprof()
417 		return element;  in mempool_alloc_noprof()
436 	/* Let's wait for someone else to return an element to @pool */  in mempool_alloc_noprof()
454  * mempool_alloc_preallocated - allocate an element from preallocated elements
460  * an element from the preallocated elements. It does not sleep and immediately
463  * Return: pointer to the allocated element or %NULL if no elements are
468 	void *element;  in mempool_alloc_preallocated()  local
473 		element = remove_element(pool);  in mempool_alloc_preallocated()
481 		kmemleak_update_trace(element);  in mempool_alloc_preallocated()
482 		return element;  in mempool_alloc_preallocated()
491  * mempool_free - return an element to the pool.
492  * @element:   pool element pointer.
498 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
502 	if (unlikely(element == NULL))  in mempool_free()
507 	 * for @element and the following @pool->curr_nr.  This ensures  in mempool_free()
509 	 * allocation of @element.  This is necessary for fringe cases  in mempool_free()
510 	 * where @element was passed to this task without going through  in mempool_free()
528 	 * allocation of @element, any task which decremented curr_nr below  in mempool_free()
531 	 * to min_nr after the allocation of @element, the elements  in mempool_free()
541 			add_element(pool, element);  in mempool_free()
548 	pool->free(element, pool->pool_data);  in mempool_free()
563 void mempool_free_slab(void *element, void *pool_data)  in mempool_free_slab()  argument
566 	kmem_cache_free(mem, element);  in mempool_free_slab()
581 void mempool_kfree(void *element, void *pool_data)  in mempool_kfree()  argument
583 	kfree(element);  in mempool_kfree()
594 void mempool_kvfree(void *element, void *pool_data)  in mempool_kvfree()  argument
596 	kvfree(element);  in mempool_kvfree()
611 void mempool_free_pages(void *element, void *pool_data)  in mempool_free_pages()  argument
614 	__free_pages(element, order);  in mempool_free_pages()