201 		for_each_member_device_rcu(c, ca, NULL)  in bch2_dev_to_fs()
202 			if (ca->disk_sb.bdev && ca->disk_sb.bdev->bd_dev == dev) {  in bch2_dev_to_fs()
300 	for_each_member_device(c, ca)  in __bch2_fs_read_only()
301 		bch2_dev_allocator_remove(c, ca);  in __bch2_fs_read_only()
486 	for_each_rw_member(c, ca)  in __bch2_fs_read_write()
487 		bch2_dev_allocator_add(c, ca);  in __bch2_fs_read_write()
637 	for_each_member_device(c, ca)  in __bch2_fs_stop()
638 		bch2_dev_unlink(ca);  in __bch2_fs_stop()
657 	for_each_member_device(c, ca)  in __bch2_fs_stop()
658 		cancel_work_sync(&ca->io_error_work);  in __bch2_fs_stop()
675 		struct bch_dev *ca = rcu_dereference_protected(c->devs[i], true);  in bch2_fs_free()  local
677 		if (ca) {  in bch2_fs_free()
678 			EBUG_ON(atomic_long_read(&ca->ref) != 1);  in bch2_fs_free()
679 			bch2_free_super(&ca->disk_sb);  in bch2_fs_free()
680 			bch2_dev_free(ca);  in bch2_fs_free()
729 	for_each_member_device(c, ca) {  in bch2_fs_online()
730 		ret = bch2_dev_sysfs_online(c, ca);  in bch2_fs_online()
733 			bch2_dev_put(ca);  in bch2_fs_online()
1022 	for_each_online_member(c, ca)  in bch2_fs_start()
1023 		bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx)->last_mount = cpu_to_le64(now);  in bch2_fs_start()
1035 	for_each_rw_member(c, ca)  in bch2_fs_start()
1036 		bch2_dev_allocator_add(c, ca);  in bch2_fs_start()
1187 	struct bch_dev *ca = container_of(kobj, struct bch_dev, kobj);  in bch2_dev_release()  local
1189 	kfree(ca);  in bch2_dev_release()
1192 static void bch2_dev_free(struct bch_dev *ca)  in bch2_dev_free()  argument
1194 	cancel_work_sync(&ca->io_error_work);  in bch2_dev_free()
1196 	bch2_dev_unlink(ca);  in bch2_dev_free()
1198 	if (ca->kobj.state_in_sysfs)  in bch2_dev_free()
1199 		kobject_del(&ca->kobj);  in bch2_dev_free()
1201 	bch2_free_super(&ca->disk_sb);  in bch2_dev_free()
1202 	bch2_dev_allocator_background_exit(ca);  in bch2_dev_free()
1203 	bch2_dev_journal_exit(ca);  in bch2_dev_free()
1205 	free_percpu(ca->io_done);  in bch2_dev_free()
1206 	bch2_dev_buckets_free(ca);  in bch2_dev_free()
1207 	kfree(ca->sb_read_scratch);  in bch2_dev_free()
1209 	bch2_time_stats_quantiles_exit(&ca->io_latency[WRITE]);  in bch2_dev_free()
1210 	bch2_time_stats_quantiles_exit(&ca->io_latency[READ]);  in bch2_dev_free()
1212 	percpu_ref_exit(&ca->io_ref);  in bch2_dev_free()
1214 	percpu_ref_exit(&ca->ref);  in bch2_dev_free()
1216 	kobject_put(&ca->kobj);  in bch2_dev_free()
1219 static void __bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca)  in __bch2_dev_offline()  argument
1224 	if (percpu_ref_is_zero(&ca->io_ref))  in __bch2_dev_offline()
1227 	__bch2_dev_read_only(c, ca);  in __bch2_dev_offline()
1229 	reinit_completion(&ca->io_ref_completion);  in __bch2_dev_offline()
1230 	percpu_ref_kill(&ca->io_ref);  in __bch2_dev_offline()
1231 	wait_for_completion(&ca->io_ref_completion);  in __bch2_dev_offline()
1233 	bch2_dev_unlink(ca);  in __bch2_dev_offline()
1235 	bch2_free_super(&ca->disk_sb);  in __bch2_dev_offline()
1236 	bch2_dev_journal_exit(ca);  in __bch2_dev_offline()
1242 	struct bch_dev *ca = container_of(ref, struct bch_dev, ref);  in bch2_dev_ref_complete()  local
1244 	complete(&ca->ref_completion);  in bch2_dev_ref_complete()
1250 	struct bch_dev *ca = container_of(ref, struct bch_dev, io_ref);  in bch2_dev_io_ref_complete()  local
1252 	complete(&ca->io_ref_completion);  in bch2_dev_io_ref_complete()
1255 static void bch2_dev_unlink(struct bch_dev *ca)  in bch2_dev_unlink()  argument
1267 	if (ca->kobj.state_in_sysfs &&  in bch2_dev_unlink()
1268 	    ca->disk_sb.bdev &&  in bch2_dev_unlink()
1269 	    (b = bdev_kobj(ca->disk_sb.bdev))->state_in_sysfs) {  in bch2_dev_unlink()
1271 		sysfs_remove_link(&ca->kobj, "block");  in bch2_dev_unlink()
1275 static int bch2_dev_sysfs_online(struct bch_fs *c, struct bch_dev *ca)  in bch2_dev_sysfs_online()  argument
1282 	if (!ca->kobj.state_in_sysfs) {  in bch2_dev_sysfs_online()
1283 		ret = kobject_add(&ca->kobj, &c->kobj,  in bch2_dev_sysfs_online()
1284 				  "dev-%u", ca->dev_idx);  in bch2_dev_sysfs_online()
1289 	if (ca->disk_sb.bdev) {  in bch2_dev_sysfs_online()
1290 		struct kobject *block = bdev_kobj(ca->disk_sb.bdev);  in bch2_dev_sysfs_online()
1292 		ret = sysfs_create_link(block, &ca->kobj, "bcachefs");  in bch2_dev_sysfs_online()
1296 		ret = sysfs_create_link(&ca->kobj, block, "block");  in bch2_dev_sysfs_online()
1307 	struct bch_dev *ca;  in __bch2_dev_alloc()  local
1310 	ca = kzalloc(sizeof(*ca), GFP_KERNEL);  in __bch2_dev_alloc()
1311 	if (!ca)  in __bch2_dev_alloc()
1314 	kobject_init(&ca->kobj, &bch2_dev_ktype);  in __bch2_dev_alloc()
1315 	init_completion(&ca->ref_completion);  in __bch2_dev_alloc()
1316 	init_completion(&ca->io_ref_completion);  in __bch2_dev_alloc()
1318 	INIT_WORK(&ca->io_error_work, bch2_io_error_work);  in __bch2_dev_alloc()
1320 	bch2_time_stats_quantiles_init(&ca->io_latency[READ]);  in __bch2_dev_alloc()
1321 	bch2_time_stats_quantiles_init(&ca->io_latency[WRITE]);  in __bch2_dev_alloc()
1323 	ca->mi = bch2_mi_to_cpu(member);  in __bch2_dev_alloc()
1326 		atomic64_set(&ca->errors[i], le64_to_cpu(member->errors[i]));  in __bch2_dev_alloc()
1328 	ca->uuid = member->uuid;  in __bch2_dev_alloc()
1330 	ca->nr_btree_reserve = DIV_ROUND_UP(BTREE_NODE_RESERVE,  in __bch2_dev_alloc()
1331 			     ca->mi.bucket_size / btree_sectors(c));  in __bch2_dev_alloc()
1334 	if (percpu_ref_init(&ca->ref, bch2_dev_ref_complete, 0, GFP_KERNEL))  in __bch2_dev_alloc()
1337 	atomic_long_set(&ca->ref, 1);  in __bch2_dev_alloc()
1340 	bch2_dev_allocator_background_init(ca);  in __bch2_dev_alloc()
1342 	if (percpu_ref_init(&ca->io_ref, bch2_dev_io_ref_complete,  in __bch2_dev_alloc()
1344 	    !(ca->sb_read_scratch = kmalloc(BCH_SB_READ_SCRATCH_BUF_SIZE, GFP_KERNEL)) ||  in __bch2_dev_alloc()
1345 	    bch2_dev_buckets_alloc(c, ca) ||  in __bch2_dev_alloc()
1346 	    !(ca->io_done	= alloc_percpu(*ca->io_done)))  in __bch2_dev_alloc()
1349 	return ca;  in __bch2_dev_alloc()
1351 	bch2_dev_free(ca);  in __bch2_dev_alloc()
1355 static void bch2_dev_attach(struct bch_fs *c, struct bch_dev *ca,  in bch2_dev_attach()  argument
1358 	ca->dev_idx = dev_idx;  in bch2_dev_attach()
1359 	__set_bit(ca->dev_idx, ca->self.d);  in bch2_dev_attach()
1360 	scnprintf(ca->name, sizeof(ca->name), "dev-%u", dev_idx);  in bch2_dev_attach()
1362 	ca->fs = c;  in bch2_dev_attach()
1363 	rcu_assign_pointer(c->devs[ca->dev_idx], ca);  in bch2_dev_attach()
1365 	if (bch2_dev_sysfs_online(c, ca))  in bch2_dev_attach()
1372 	struct bch_dev *ca = NULL;  in bch2_dev_alloc()  local
1377 	ca = __bch2_dev_alloc(c, &member);  in bch2_dev_alloc()
1378 	if (!ca)  in bch2_dev_alloc()
1381 	ca->fs = c;  in bch2_dev_alloc()
1383 	bch2_dev_attach(c, ca, dev_idx);  in bch2_dev_alloc()
1389 static int __bch2_dev_attach_bdev(struct bch_dev *ca, struct bch_sb_handle *sb)  in __bch2_dev_attach_bdev()  argument
1393 	if (bch2_dev_is_online(ca)) {  in __bch2_dev_attach_bdev()
1394 		bch_err(ca, "already have device online in slot %u",  in __bch2_dev_attach_bdev()
1400 	    ca->mi.bucket_size * ca->mi.nbuckets) {  in __bch2_dev_attach_bdev()
1401 		bch_err(ca, "cannot online: device too small");  in __bch2_dev_attach_bdev()
1405 	BUG_ON(!percpu_ref_is_zero(&ca->io_ref));  in __bch2_dev_attach_bdev()
1407 	ret = bch2_dev_journal_init(ca, sb->sb);  in __bch2_dev_attach_bdev()
1412 	ca->disk_sb = *sb;  in __bch2_dev_attach_bdev()
1415 	ca->dev = ca->disk_sb.bdev->bd_dev;  in __bch2_dev_attach_bdev()
1417 	percpu_ref_reinit(&ca->io_ref);  in __bch2_dev_attach_bdev()
1424 	struct bch_dev *ca;  in bch2_dev_attach_bdev()  local
1435 	ca = bch2_dev_locked(c, sb->sb->dev_idx);  in bch2_dev_attach_bdev()
1437 	ret = __bch2_dev_attach_bdev(ca, sb);  in bch2_dev_attach_bdev()
1441 	bch2_dev_sysfs_online(c, ca);  in bch2_dev_attach_bdev()
1444 	prt_bdevname(&name, ca->disk_sb.bdev);  in bch2_dev_attach_bdev()
1448 	strscpy(ca->name, name.buf, sizeof(ca->name));  in bch2_dev_attach_bdev()
1467 bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca,  in bch2_dev_state_allowed()  argument
1479 		if (ca->mi.state != BCH_MEMBER_STATE_rw)  in bch2_dev_state_allowed()
1484 			if (ca2 != ca)  in bch2_dev_state_allowed()
1497 		if (ca->mi.state != BCH_MEMBER_STATE_rw &&  in bch2_dev_state_allowed()
1498 		    ca->mi.state != BCH_MEMBER_STATE_ro)  in bch2_dev_state_allowed()
1503 		__clear_bit(ca->dev_idx, new_online_devs.d);  in bch2_dev_state_allowed()
1513 	struct bch_dev *ca;  in bch2_fs_may_start()  local
1530 			ca = bch2_dev_locked(c, i);  in bch2_fs_may_start()
1532 			if (!bch2_dev_is_online(ca) &&  in bch2_fs_may_start()
1533 			    (ca->mi.state == BCH_MEMBER_STATE_rw ||  in bch2_fs_may_start()
1534 			     ca->mi.state == BCH_MEMBER_STATE_ro)) {  in bch2_fs_may_start()
1545 static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca)  in __bch2_dev_read_only()  argument
1550 	bch2_dev_allocator_remove(c, ca);  in __bch2_dev_read_only()
1552 	bch2_dev_journal_stop(&c->journal, ca);  in __bch2_dev_read_only()
1555 static void __bch2_dev_read_write(struct bch_fs *c, struct bch_dev *ca)  in __bch2_dev_read_write()  argument
1559 	BUG_ON(ca->mi.state != BCH_MEMBER_STATE_rw);  in __bch2_dev_read_write()
1561 	bch2_dev_allocator_add(c, ca);  in __bch2_dev_read_write()
1563 	bch2_dev_do_discards(ca);  in __bch2_dev_read_write()
1566 int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,  in __bch2_dev_set_state()  argument
1572 	if (ca->mi.state == new_state)  in __bch2_dev_set_state()
1575 	if (!bch2_dev_state_allowed(c, ca, new_state, flags))  in __bch2_dev_set_state()
1579 		__bch2_dev_read_only(c, ca);  in __bch2_dev_set_state()
1581 	bch_notice(ca, "%s", bch2_member_states[new_state]);  in __bch2_dev_set_state()
1584 	m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);  in __bch2_dev_set_state()
1590 		__bch2_dev_read_write(c, ca);  in __bch2_dev_set_state()
1597 int bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,  in bch2_dev_set_state()  argument
1603 	ret = __bch2_dev_set_state(c, ca, new_state, flags);  in bch2_dev_set_state()
1611 int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)  in bch2_dev_remove()  argument
1614 	unsigned dev_idx = ca->dev_idx, data;  in bch2_dev_remove()
1620 	 * We consume a reference to ca->ref, regardless of whether we succeed  in bch2_dev_remove()
1623 	bch2_dev_put(ca);  in bch2_dev_remove()
1625 	if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) {  in bch2_dev_remove()
1626 		bch_err(ca, "Cannot remove without losing data");  in bch2_dev_remove()
1631 	__bch2_dev_read_only(c, ca);  in bch2_dev_remove()
1633 	ret = bch2_dev_data_drop(c, ca->dev_idx, flags);  in bch2_dev_remove()
1634 	bch_err_msg(ca, ret, "bch2_dev_data_drop()");  in bch2_dev_remove()
1638 	ret = bch2_dev_remove_alloc(c, ca);  in bch2_dev_remove()
1639 	bch_err_msg(ca, ret, "bch2_dev_remove_alloc()");  in bch2_dev_remove()
1653 	ret = bch2_journal_flush_device_pins(&c->journal, ca->dev_idx);  in bch2_dev_remove()
1654 	bch_err_msg(ca, ret, "bch2_journal_flush_device_pins()");  in bch2_dev_remove()
1659 	bch_err_msg(ca, ret, "bch2_journal_flush()");  in bch2_dev_remove()
1664 	bch_err_msg(ca, ret, "bch2_replicas_gc2()");  in bch2_dev_remove()
1668 	data = bch2_dev_has_data(c, ca);  in bch2_dev_remove()
1673 		bch_err(ca, "Remove failed, still has data (%s)", data_has.buf);  in bch2_dev_remove()
1679 	__bch2_dev_offline(c, ca);  in bch2_dev_remove()
1682 	rcu_assign_pointer(c->devs[ca->dev_idx], NULL);  in bch2_dev_remove()
1686 	percpu_ref_kill(&ca->ref);  in bch2_dev_remove()
1688 	ca->dying = true;  in bch2_dev_remove()
1689 	bch2_dev_put(ca);  in bch2_dev_remove()
1691 	wait_for_completion(&ca->ref_completion);  in bch2_dev_remove()
1693 	bch2_dev_free(ca);  in bch2_dev_remove()
1709 	if (ca->mi.state == BCH_MEMBER_STATE_rw &&  in bch2_dev_remove()
1710 	    !percpu_ref_is_zero(&ca->io_ref))  in bch2_dev_remove()
1711 		__bch2_dev_read_write(c, ca);  in bch2_dev_remove()
1721 	struct bch_dev *ca = NULL;  in bch2_dev_add()  local
1745 	ca = __bch2_dev_alloc(c, &dev_mi);  in bch2_dev_add()
1746 	if (!ca) {  in bch2_dev_add()
1751 	ret = __bch2_dev_attach_bdev(ca, &sb);  in bch2_dev_add()
1758 	ret = bch2_sb_from_fs(c, ca);  in bch2_dev_add()
1778 	ca->disk_sb.sb->dev_idx	= dev_idx;  in bch2_dev_add()
1779 	bch2_dev_attach(c, ca, dev_idx);  in bch2_dev_add()
1782 		ret = __bch2_dev_group_set(c, ca, label.buf);  in bch2_dev_add()
1791 	ret = bch2_dev_usage_init(ca, false);  in bch2_dev_add()
1795 	ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional);  in bch2_dev_add()
1796 	bch_err_msg(ca, ret, "marking new superblock");  in bch2_dev_add()
1801 	bch_err_msg(ca, ret, "initializing free space");  in bch2_dev_add()
1805 	if (ca->mi.state == BCH_MEMBER_STATE_rw)  in bch2_dev_add()
1806 		__bch2_dev_read_write(c, ca);  in bch2_dev_add()
1808 	ret = bch2_dev_journal_alloc(ca, false);  in bch2_dev_add()
1824 	if (ca)  in bch2_dev_add()
1825 		bch2_dev_free(ca);  in bch2_dev_add()
1830 	ca = NULL;  in bch2_dev_add()
1839 	struct bch_dev *ca;  in bch2_dev_online()  local
1862 	ca = bch2_dev_locked(c, dev_idx);  in bch2_dev_online()
1864 	ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional);  in bch2_dev_online()
1869 	if (ca->mi.state == BCH_MEMBER_STATE_rw)  in bch2_dev_online()
1870 		__bch2_dev_read_write(c, ca);  in bch2_dev_online()
1872 	if (!ca->mi.freespace_initialized) {  in bch2_dev_online()
1873 		ret = bch2_dev_freespace_init(c, ca, 0, ca->mi.nbuckets);  in bch2_dev_online()
1874 		bch_err_msg(ca, ret, "initializing free space");  in bch2_dev_online()
1879 	if (!ca->journal.nr) {  in bch2_dev_online()
1880 		ret = bch2_dev_journal_alloc(ca, false);  in bch2_dev_online()
1881 		bch_err_msg(ca, ret, "allocating journal");  in bch2_dev_online()
1887 	bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx)->last_mount =  in bch2_dev_online()
1900 int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags)  in bch2_dev_offline()  argument
1904 	if (!bch2_dev_is_online(ca)) {  in bch2_dev_offline()
1905 		bch_err(ca, "Already offline");  in bch2_dev_offline()
1910 	if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) {  in bch2_dev_offline()
1911 		bch_err(ca, "Cannot offline required disk");  in bch2_dev_offline()
1916 	__bch2_dev_offline(c, ca);  in bch2_dev_offline()
1922 int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)  in bch2_dev_resize()  argument
1929 	old_nbuckets = ca->mi.nbuckets;  in bch2_dev_resize()
1931 	if (nbuckets < ca->mi.nbuckets) {  in bch2_dev_resize()
1932 		bch_err(ca, "Cannot shrink yet");  in bch2_dev_resize()
1938 		bch_err(ca, "New device size too big (%llu greater than max %u)",  in bch2_dev_resize()
1944 	if (bch2_dev_is_online(ca) &&  in bch2_dev_resize()
1945 	    get_capacity(ca->disk_sb.bdev->bd_disk) <  in bch2_dev_resize()
1946 	    ca->mi.bucket_size * nbuckets) {  in bch2_dev_resize()
1947 		bch_err(ca, "New size larger than device");  in bch2_dev_resize()
1952 	ret = bch2_dev_buckets_resize(c, ca, nbuckets);  in bch2_dev_resize()
1953 	bch_err_msg(ca, ret, "resizing buckets");  in bch2_dev_resize()
1957 	ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional);  in bch2_dev_resize()
1962 	m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);  in bch2_dev_resize()
1968 	if (ca->mi.freespace_initialized) {  in bch2_dev_resize()
1971 			.dev_data_type.dev = ca->dev_idx,  in bch2_dev_resize()
1976 		ret   = bch2_trans_commit_do(ca->fs, NULL, NULL, 0,  in bch2_dev_resize()
1978 			bch2_dev_freespace_init(c, ca, old_nbuckets, nbuckets);  in bch2_dev_resize()
1989 /* return with ref on ca->ref: */
1995 	for_each_member_device(c, ca)  in bch2_dev_lookup()
1996 		if (!strcmp(name, ca->name))  in bch2_dev_lookup()
1997 			return ca;  in bch2_dev_lookup()