76 uint8_t bch_inc_gen(struct cache *ca, struct bucket *b)  in bch_inc_gen()  argument
80 	ca->set->need_gc = max(ca->set->need_gc, bucket_gc_gen(b));  in bch_inc_gen()
81 	WARN_ON_ONCE(ca->set->need_gc > BUCKET_GC_GEN_MAX);  in bch_inc_gen()
88 	struct cache *ca;  in bch_rescale_priorities()  local
106 	ca = c->cache;  in bch_rescale_priorities()
107 	for_each_bucket(b, ca)  in bch_rescale_priorities()
130 bool bch_can_invalidate_bucket(struct cache *ca, struct bucket *b)  in bch_can_invalidate_bucket()  argument
132 	return (ca->set->gc_mark_valid || b->reclaimable_in_gc) &&  in bch_can_invalidate_bucket()
137 void __bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)  in __bch_invalidate_one_bucket()  argument
139 	lockdep_assert_held(&ca->set->bucket_lock);  in __bch_invalidate_one_bucket()
143 		trace_bcache_invalidate(ca, b - ca->buckets);  in __bch_invalidate_one_bucket()
145 	bch_inc_gen(ca, b);  in __bch_invalidate_one_bucket()
151 static void bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)  in bch_invalidate_one_bucket()  argument
153 	__bch_invalidate_one_bucket(ca, b);  in bch_invalidate_one_bucket()
155 	fifo_push(&ca->free_inc, b - ca->buckets);  in bch_invalidate_one_bucket()
167 static inline unsigned int new_bucket_prio(struct cache *ca, struct bucket *b)  in new_bucket_prio()  argument
169 	unsigned int min_prio = (INITIAL_PRIO - ca->set->min_prio) / 8;  in new_bucket_prio()
171 	return (b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b);  in new_bucket_prio()
178 	struct cache *ca = args;  in new_bucket_max_cmp()  local
180 	return new_bucket_prio(ca, *lhs) > new_bucket_prio(ca, *rhs);  in new_bucket_max_cmp()
187 	struct cache *ca = args;  in new_bucket_min_cmp()  local
189 	return new_bucket_prio(ca, *lhs) < new_bucket_prio(ca, *rhs);  in new_bucket_min_cmp()
192 static void invalidate_buckets_lru(struct cache *ca)  in invalidate_buckets_lru()  argument
204 	ca->heap.nr = 0;  in invalidate_buckets_lru()
206 	for_each_bucket(b, ca) {  in invalidate_buckets_lru()
207 		if (!bch_can_invalidate_bucket(ca, b))  in invalidate_buckets_lru()
210 		if (!min_heap_full(&ca->heap))  in invalidate_buckets_lru()
211 			min_heap_push(&ca->heap, &b, &bucket_max_cmp_callback, ca);  in invalidate_buckets_lru()
212 		else if (!new_bucket_max_cmp(&b, min_heap_peek(&ca->heap), ca)) {  in invalidate_buckets_lru()
213 			ca->heap.data[0] = b;  in invalidate_buckets_lru()
214 			min_heap_sift_down(&ca->heap, 0, &bucket_max_cmp_callback, ca);  in invalidate_buckets_lru()
218 	min_heapify_all(&ca->heap, &bucket_min_cmp_callback, ca);  in invalidate_buckets_lru()
220 	while (!fifo_full(&ca->free_inc)) {  in invalidate_buckets_lru()
221 		if (!ca->heap.nr) {  in invalidate_buckets_lru()
226 			ca->invalidate_needs_gc = 1;  in invalidate_buckets_lru()
227 			wake_up_gc(ca->set);  in invalidate_buckets_lru()
230 		b = min_heap_peek(&ca->heap)[0];  in invalidate_buckets_lru()
231 		min_heap_pop(&ca->heap, &bucket_min_cmp_callback, ca);  in invalidate_buckets_lru()
233 		bch_invalidate_one_bucket(ca, b);  in invalidate_buckets_lru()
237 static void invalidate_buckets_fifo(struct cache *ca)  in invalidate_buckets_fifo()  argument
242 	while (!fifo_full(&ca->free_inc)) {  in invalidate_buckets_fifo()
243 		if (ca->fifo_last_bucket <  ca->sb.first_bucket ||  in invalidate_buckets_fifo()
244 		    ca->fifo_last_bucket >= ca->sb.nbuckets)  in invalidate_buckets_fifo()
245 			ca->fifo_last_bucket = ca->sb.first_bucket;  in invalidate_buckets_fifo()
247 		b = ca->buckets + ca->fifo_last_bucket++;  in invalidate_buckets_fifo()
249 		if (bch_can_invalidate_bucket(ca, b))  in invalidate_buckets_fifo()
250 			bch_invalidate_one_bucket(ca, b);  in invalidate_buckets_fifo()
252 		if (++checked >= ca->sb.nbuckets) {  in invalidate_buckets_fifo()
253 			ca->invalidate_needs_gc = 1;  in invalidate_buckets_fifo()
254 			wake_up_gc(ca->set);  in invalidate_buckets_fifo()
260 static void invalidate_buckets_random(struct cache *ca)  in invalidate_buckets_random()  argument
265 	while (!fifo_full(&ca->free_inc)) {  in invalidate_buckets_random()
270 		n %= (size_t) (ca->sb.nbuckets - ca->sb.first_bucket);  in invalidate_buckets_random()
271 		n += ca->sb.first_bucket;  in invalidate_buckets_random()
273 		b = ca->buckets + n;  in invalidate_buckets_random()
275 		if (bch_can_invalidate_bucket(ca, b))  in invalidate_buckets_random()
276 			bch_invalidate_one_bucket(ca, b);  in invalidate_buckets_random()
278 		if (++checked >= ca->sb.nbuckets / 2) {  in invalidate_buckets_random()
279 			ca->invalidate_needs_gc = 1;  in invalidate_buckets_random()
280 			wake_up_gc(ca->set);  in invalidate_buckets_random()
286 static void invalidate_buckets(struct cache *ca)  in invalidate_buckets()  argument
288 	BUG_ON(ca->invalidate_needs_gc);  in invalidate_buckets()
290 	switch (CACHE_REPLACEMENT(&ca->sb)) {  in invalidate_buckets()
292 		invalidate_buckets_lru(ca);  in invalidate_buckets()
295 		invalidate_buckets_fifo(ca);  in invalidate_buckets()
298 		invalidate_buckets_random(ca);  in invalidate_buckets()
303 #define allocator_wait(ca, cond)					\  argument
310 		mutex_unlock(&(ca)->set->bucket_lock);			\
312 		    test_bit(CACHE_SET_IO_DISABLE, &ca->set->flags)) {	\
318 		mutex_lock(&(ca)->set->bucket_lock);			\
323 static int bch_allocator_push(struct cache *ca, long bucket)  in bch_allocator_push()  argument
328 	if (fifo_push(&ca->free[RESERVE_PRIO], bucket))  in bch_allocator_push()
332 		if (fifo_push(&ca->free[i], bucket))  in bch_allocator_push()
340 	struct cache *ca = arg;  in bch_allocator_thread()  local
342 	mutex_lock(&ca->set->bucket_lock);  in bch_allocator_thread()
353 			if (!fifo_pop(&ca->free_inc, bucket))  in bch_allocator_thread()
356 			if (ca->discard) {  in bch_allocator_thread()
357 				mutex_unlock(&ca->set->bucket_lock);  in bch_allocator_thread()
358 				blkdev_issue_discard(ca->bdev,  in bch_allocator_thread()
359 					bucket_to_sector(ca->set, bucket),  in bch_allocator_thread()
360 					ca->sb.bucket_size, GFP_KERNEL);  in bch_allocator_thread()
361 				mutex_lock(&ca->set->bucket_lock);  in bch_allocator_thread()
364 			allocator_wait(ca, bch_allocator_push(ca, bucket));  in bch_allocator_thread()
365 			wake_up(&ca->set->btree_cache_wait);  in bch_allocator_thread()
366 			wake_up(&ca->set->bucket_wait);  in bch_allocator_thread()
376 		allocator_wait(ca, !ca->invalidate_needs_gc);  in bch_allocator_thread()
377 		invalidate_buckets(ca);  in bch_allocator_thread()
383 		allocator_wait(ca, !atomic_read(&ca->set->prio_blocked));  in bch_allocator_thread()
384 		if (CACHE_SYNC(&ca->sb)) {  in bch_allocator_thread()
396 			if (!fifo_full(&ca->free_inc))  in bch_allocator_thread()
399 			if (bch_prio_write(ca, false) < 0) {  in bch_allocator_thread()
400 				ca->invalidate_needs_gc = 1;  in bch_allocator_thread()
401 				wake_up_gc(ca->set);  in bch_allocator_thread()
412 long bch_bucket_alloc(struct cache *ca, unsigned int reserve, bool wait)  in bch_bucket_alloc()  argument
420 	if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &ca->set->flags)))  in bch_bucket_alloc()
424 	if (fifo_pop(&ca->free[RESERVE_NONE], r) ||  in bch_bucket_alloc()
425 	    fifo_pop(&ca->free[reserve], r))  in bch_bucket_alloc()
429 		trace_bcache_alloc_fail(ca, reserve);  in bch_bucket_alloc()
434 		prepare_to_wait(&ca->set->bucket_wait, &w,  in bch_bucket_alloc()
437 		mutex_unlock(&ca->set->bucket_lock);  in bch_bucket_alloc()
439 		mutex_lock(&ca->set->bucket_lock);  in bch_bucket_alloc()
440 	} while (!fifo_pop(&ca->free[RESERVE_NONE], r) &&  in bch_bucket_alloc()
441 		 !fifo_pop(&ca->free[reserve], r));  in bch_bucket_alloc()
443 	finish_wait(&ca->set->bucket_wait, &w);  in bch_bucket_alloc()
445 	if (ca->alloc_thread)  in bch_bucket_alloc()
446 		wake_up_process(ca->alloc_thread);  in bch_bucket_alloc()
448 	trace_bcache_alloc(ca, reserve);  in bch_bucket_alloc()
450 	if (expensive_debug_checks(ca->set)) {  in bch_bucket_alloc()
455 		for (iter = 0; iter < prio_buckets(ca) * 2; iter++)  in bch_bucket_alloc()
456 			BUG_ON(ca->prio_buckets[iter] == (uint64_t) r);  in bch_bucket_alloc()
459 			fifo_for_each(i, &ca->free[j], iter)  in bch_bucket_alloc()
461 		fifo_for_each(i, &ca->free_inc, iter)  in bch_bucket_alloc()
465 	b = ca->buckets + r;  in bch_bucket_alloc()
469 	SET_GC_SECTORS_USED(b, ca->sb.bucket_size);  in bch_bucket_alloc()
481 	if (ca->set->avail_nbuckets > 0) {  in bch_bucket_alloc()
482 		ca->set->avail_nbuckets--;  in bch_bucket_alloc()
483 		bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);  in bch_bucket_alloc()
489 void __bch_bucket_free(struct cache *ca, struct bucket *b)  in __bch_bucket_free()  argument
494 	if (ca->set->avail_nbuckets < ca->set->nbuckets) {  in __bch_bucket_free()
495 		ca->set->avail_nbuckets++;  in __bch_bucket_free()
496 		bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);  in __bch_bucket_free()
511 	struct cache *ca;  in __bch_bucket_alloc_set()  local
522 	ca = c->cache;  in __bch_bucket_alloc_set()
523 	b = bch_bucket_alloc(ca, reserve, wait);  in __bch_bucket_alloc_set()
527 	k->ptr[0] = MAKE_PTR(ca->buckets[b].gen,  in __bch_bucket_alloc_set()
529 			     ca->sb.nr_this_dev);  in __bch_bucket_alloc_set()
743 int bch_cache_allocator_start(struct cache *ca)  in bch_cache_allocator_start()  argument
746 					    ca, "bcache_allocator");  in bch_cache_allocator_start()
750 	ca->alloc_thread = k;  in bch_cache_allocator_start()