Lines Matching full:bitmap
3 * bitmap.c two-level bitmap (C) Peter T. Breuer ([email protected]) 2003
5 * bitmap_create - sets up the bitmap structure
6 * bitmap_destroy - destroys the bitmap structure
9 * - added disk storage for bitmap
10 * - changes to allow various bitmap chunk sizes
33 #include "md-bitmap.h"
36 /* version 4 insists the bitmap is in little-endian order
45 * in-memory bitmap:
123 * bitmap structures:
126 /* the in-memory bitmap is represented by bitmap_pages */
148 /* the main bitmap structure - one per mddev */
149 struct bitmap { struct
154 /* total number of pages in the bitmap */
164 struct mddev *mddev; /* the md device that the bitmap is for */ argument
172 /* cached copy of the bitmap file superblock */ argument
181 /* total bytes in the bitmap */
194 * the bitmap daemon - periodically wakes up and sweeps the bitmap argument
199 * when we lasted called end_sync to update bitmap with resync argument
204 /* pending writes to the bitmap file */ argument
215 static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, argument
218 static inline char *bmname(struct bitmap *bitmap) in bmname() argument
220 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX"; in bmname()
223 static bool __bitmap_enabled(struct bitmap *bitmap) in __bitmap_enabled() argument
225 return bitmap->storage.filemap && in __bitmap_enabled()
226 !test_bit(BITMAP_STALE, &bitmap->flags); in __bitmap_enabled()
231 struct bitmap *bitmap = mddev->bitmap; in bitmap_enabled() local
233 if (!bitmap) in bitmap_enabled()
236 return __bitmap_enabled(bitmap); in bitmap_enabled()
249 static int md_bitmap_checkpage(struct bitmap_counts *bitmap, in md_bitmap_checkpage() argument
251 __releases(bitmap->lock) in md_bitmap_checkpage()
252 __acquires(bitmap->lock) in md_bitmap_checkpage()
256 WARN_ON_ONCE(page >= bitmap->pages); in md_bitmap_checkpage()
257 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */ in md_bitmap_checkpage()
260 if (bitmap->bp[page].map) /* page is already allocated, just return */ in md_bitmap_checkpage()
268 spin_unlock_irq(&bitmap->lock); in md_bitmap_checkpage()
283 spin_lock_irq(&bitmap->lock); in md_bitmap_checkpage()
286 pr_debug("md/bitmap: map page allocation failed, hijacking\n"); in md_bitmap_checkpage()
292 if (!bitmap->bp[page].map) in md_bitmap_checkpage()
293 bitmap->bp[page].hijacked = 1; in md_bitmap_checkpage()
294 } else if (bitmap->bp[page].map || in md_bitmap_checkpage()
295 bitmap->bp[page].hijacked) { in md_bitmap_checkpage()
302 bitmap->bp[page].map = mappage; in md_bitmap_checkpage()
303 bitmap->missing_pages--; in md_bitmap_checkpage()
311 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page) in md_bitmap_checkfree() argument
315 if (bitmap->bp[page].count) /* page is still busy */ in md_bitmap_checkfree()
320 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */ in md_bitmap_checkfree()
321 bitmap->bp[page].hijacked = 0; in md_bitmap_checkfree()
322 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
325 ptr = bitmap->bp[page].map; in md_bitmap_checkfree()
326 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
327 bitmap->missing_pages++; in md_bitmap_checkfree()
333 * bitmap file handling - read and write the bitmap file and its superblock
340 /* IO operations when bitmap is stored near all superblocks */
423 static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap, in __write_sb_page() argument
427 struct mddev *mddev = bitmap->mddev; in __write_sb_page()
428 struct bitmap_storage *store = &bitmap->storage; in __write_sb_page()
429 unsigned long num_pages = bitmap->storage.file_pages; in __write_sb_page()
453 /* Bitmap could be anywhere. */ in __write_sb_page()
458 /* DATA BITMAP METADATA */ in __write_sb_page()
461 /* bitmap runs in to metadata */ in __write_sb_page()
465 /* data runs in to bitmap */ in __write_sb_page()
468 /* METADATA BITMAP DATA */ in __write_sb_page()
471 /* bitmap runs in to data */ in __write_sb_page()
479 static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index, in write_sb_page() argument
482 struct mddev *mddev = bitmap->mddev; in write_sb_page()
488 if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) { in write_sb_page()
489 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in write_sb_page()
496 static void md_bitmap_file_kick(struct bitmap *bitmap);
499 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) in write_file_page() argument
504 atomic_inc(&bitmap->pending_writes); in write_file_page()
512 wait_event(bitmap->write_wait, in write_file_page()
513 atomic_read(&bitmap->pending_writes) == 0); in write_file_page()
518 struct bitmap *bitmap = bh->b_private; in end_bitmap_write() local
521 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in end_bitmap_write()
522 if (atomic_dec_and_test(&bitmap->pending_writes)) in end_bitmap_write()
523 wake_up(&bitmap->write_wait); in end_bitmap_write()
551 struct bitmap *bitmap, unsigned long count, struct page *page) in read_file_page() argument
559 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE, in read_file_page()
590 bh->b_private = bitmap; in read_file_page()
591 atomic_inc(&bitmap->pending_writes); in read_file_page()
600 wait_event(bitmap->write_wait, in read_file_page()
601 atomic_read(&bitmap->pending_writes)==0); in read_file_page()
602 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in read_file_page()
606 pr_err("md: bitmap read error: (%dB @ %llu): %d\n", in read_file_page()
613 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) in write_file_page() argument
617 struct bitmap *bitmap, unsigned long count, struct page *page) in read_file_page() argument
628 * bitmap file superblock operations
634 static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index, in filemap_write_page() argument
637 struct bitmap_storage *store = &bitmap->storage; in filemap_write_page()
640 if (mddev_is_clustered(bitmap->mddev)) { in filemap_write_page()
641 /* go to node bitmap area starting point */ in filemap_write_page()
646 write_file_page(bitmap, page, wait); in filemap_write_page()
648 write_sb_page(bitmap, pg_index, page, wait); in filemap_write_page()
652 * md_bitmap_wait_writes() should be called before writing any bitmap
656 static void md_bitmap_wait_writes(struct bitmap *bitmap) in md_bitmap_wait_writes() argument
658 if (bitmap->storage.file) in md_bitmap_wait_writes()
659 wait_event(bitmap->write_wait, in md_bitmap_wait_writes()
660 atomic_read(&bitmap->pending_writes)==0); in md_bitmap_wait_writes()
665 * which is safe. The relevant bitmap blocks will in md_bitmap_wait_writes()
669 md_super_wait(bitmap->mddev); in md_bitmap_wait_writes()
677 struct bitmap *bitmap = data; in bitmap_update_sb() local
679 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */ in bitmap_update_sb()
681 if (bitmap->mddev->bitmap_info.external) in bitmap_update_sb()
683 if (!bitmap->storage.sb_page) /* no superblock */ in bitmap_update_sb()
685 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_update_sb()
686 sb->events = cpu_to_le64(bitmap->mddev->events); in bitmap_update_sb()
687 if (bitmap->mddev->events < bitmap->events_cleared) in bitmap_update_sb()
689 bitmap->events_cleared = bitmap->mddev->events; in bitmap_update_sb()
690 sb->events_cleared = cpu_to_le64(bitmap->events_cleared); in bitmap_update_sb()
693 * a bitmap write error occurred but the later writes succeeded. in bitmap_update_sb()
695 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR)); in bitmap_update_sb()
697 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ); in bitmap_update_sb()
698 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind); in bitmap_update_sb()
700 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in bitmap_update_sb()
701 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize); in bitmap_update_sb()
702 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes); in bitmap_update_sb()
703 sb->sectors_reserved = cpu_to_le32(bitmap->mddev-> in bitmap_update_sb()
707 if (bitmap->storage.file) in bitmap_update_sb()
708 write_file_page(bitmap, bitmap->storage.sb_page, 1); in bitmap_update_sb()
710 write_sb_page(bitmap, bitmap->storage.sb_index, in bitmap_update_sb()
711 bitmap->storage.sb_page, 1); in bitmap_update_sb()
714 static void bitmap_print_sb(struct bitmap *bitmap) in bitmap_print_sb() argument
718 if (!bitmap || !bitmap->storage.sb_page) in bitmap_print_sb()
720 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_print_sb()
721 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap)); in bitmap_print_sb()
744 * @bitmap
747 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
748 * This function verifies 'bitmap_info' and populates the on-disk bitmap
753 static int md_bitmap_new_disk_sb(struct bitmap *bitmap) in md_bitmap_new_disk_sb() argument
758 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO); in md_bitmap_new_disk_sb()
759 if (bitmap->storage.sb_page == NULL) in md_bitmap_new_disk_sb()
761 bitmap->storage.sb_index = 0; in md_bitmap_new_disk_sb()
763 sb = kmap_local_page(bitmap->storage.sb_page); in md_bitmap_new_disk_sb()
768 chunksize = bitmap->mddev->bitmap_info.chunksize; in md_bitmap_new_disk_sb()
772 pr_warn("bitmap chunksize not a power of 2\n"); in md_bitmap_new_disk_sb()
777 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep; in md_bitmap_new_disk_sb()
783 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_new_disk_sb()
789 write_behind = bitmap->mddev->bitmap_info.max_write_behind; in md_bitmap_new_disk_sb()
793 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_new_disk_sb()
795 /* keep the array size field of the bitmap superblock up to date */ in md_bitmap_new_disk_sb()
796 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_new_disk_sb()
798 memcpy(sb->uuid, bitmap->mddev->uuid, 16); in md_bitmap_new_disk_sb()
800 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_new_disk_sb()
801 sb->state = cpu_to_le32(bitmap->flags); in md_bitmap_new_disk_sb()
802 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_new_disk_sb()
803 sb->events_cleared = cpu_to_le64(bitmap->mddev->events); in md_bitmap_new_disk_sb()
804 bitmap->mddev->bitmap_info.nodes = 0; in md_bitmap_new_disk_sb()
811 /* read the superblock from the bitmap file and initialize some bitmap fields */
812 static int md_bitmap_read_sb(struct bitmap *bitmap) in md_bitmap_read_sb() argument
824 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) { in md_bitmap_read_sb()
828 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
836 bitmap->storage.sb_page = sb_page; in md_bitmap_read_sb()
840 if (bitmap->cluster_slot >= 0) { in md_bitmap_read_sb()
841 sector_t bm_blocks = bitmap->mddev->resync_max_sectors; in md_bitmap_read_sb()
844 (bitmap->mddev->bitmap_info.chunksize >> 9)); in md_bitmap_read_sb()
849 offset = bitmap->cluster_slot * (bm_blocks << 3); in md_bitmap_read_sb()
851 bitmap->cluster_slot, offset); in md_bitmap_read_sb()
854 if (bitmap->storage.file) { in md_bitmap_read_sb()
855 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host); in md_bitmap_read_sb()
858 err = read_file_page(bitmap->storage.file, 0, in md_bitmap_read_sb()
859 bitmap, bytes, sb_page); in md_bitmap_read_sb()
861 err = read_sb_page(bitmap->mddev, offset, sb_page, 0, in md_bitmap_read_sb()
875 /* verify that the bitmap-specific fields are valid */ in md_bitmap_read_sb()
882 reason = "bitmap chunksize too small"; in md_bitmap_read_sb()
884 reason = "bitmap chunksize not a power of 2"; in md_bitmap_read_sb()
890 pr_warn("%s: invalid bitmap file superblock: %s\n", in md_bitmap_read_sb()
891 bmname(bitmap), reason); in md_bitmap_read_sb()
896 * Setup nodes/clustername only if bitmap version is in md_bitmap_read_sb()
901 strscpy(bitmap->mddev->bitmap_info.cluster_name, in md_bitmap_read_sb()
905 /* keep the array size field of the bitmap superblock up to date */ in md_bitmap_read_sb()
906 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_read_sb()
908 if (bitmap->mddev->persistent) { in md_bitmap_read_sb()
911 * bitmap's UUID and event counter to the mddev's in md_bitmap_read_sb()
913 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) { in md_bitmap_read_sb()
914 pr_warn("%s: bitmap superblock UUID mismatch\n", in md_bitmap_read_sb()
915 bmname(bitmap)); in md_bitmap_read_sb()
919 if (!nodes && (events < bitmap->mddev->events)) { in md_bitmap_read_sb()
920 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n", in md_bitmap_read_sb()
921 bmname(bitmap), events, in md_bitmap_read_sb()
922 (unsigned long long) bitmap->mddev->events); in md_bitmap_read_sb()
923 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
928 bitmap->flags |= le32_to_cpu(sb->state); in md_bitmap_read_sb()
930 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags); in md_bitmap_read_sb()
931 bitmap->events_cleared = le64_to_cpu(sb->events_cleared); in md_bitmap_read_sb()
936 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) { in md_bitmap_read_sb()
938 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
939 err = md_setup_cluster(bitmap->mddev, nodes); in md_bitmap_read_sb()
942 bmname(bitmap), err); in md_bitmap_read_sb()
945 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev); in md_bitmap_read_sb()
951 if (test_bit(BITMAP_STALE, &bitmap->flags)) in md_bitmap_read_sb()
952 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_read_sb()
953 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
954 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_read_sb()
955 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_read_sb()
956 bitmap->mddev->bitmap_info.nodes = nodes; in md_bitmap_read_sb()
957 if (bitmap->mddev->bitmap_info.space == 0 || in md_bitmap_read_sb()
958 bitmap->mddev->bitmap_info.space > sectors_reserved) in md_bitmap_read_sb()
959 bitmap->mddev->bitmap_info.space = sectors_reserved; in md_bitmap_read_sb()
961 bitmap_print_sb(bitmap); in md_bitmap_read_sb()
962 if (bitmap->cluster_slot < 0) in md_bitmap_read_sb()
963 md_cluster_stop(bitmap->mddev); in md_bitmap_read_sb()
969 * general bitmap file operations
973 * on-disk bitmap:
975 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
1087 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
1091 static void md_bitmap_file_kick(struct bitmap *bitmap) in md_bitmap_file_kick() argument
1093 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) { in md_bitmap_file_kick()
1094 bitmap_update_sb(bitmap); in md_bitmap_file_kick()
1096 if (bitmap->storage.file) { in md_bitmap_file_kick()
1097 pr_warn("%s: kicking failed bitmap file %pD4 from array!\n", in md_bitmap_file_kick()
1098 bmname(bitmap), bitmap->storage.file); in md_bitmap_file_kick()
1101 pr_warn("%s: disabling internal bitmap due to errors\n", in md_bitmap_file_kick()
1102 bmname(bitmap)); in md_bitmap_file_kick()
1113 static inline void set_page_attr(struct bitmap *bitmap, int pnum, in set_page_attr() argument
1116 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in set_page_attr()
1119 static inline void clear_page_attr(struct bitmap *bitmap, int pnum, in clear_page_attr() argument
1122 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in clear_page_attr()
1125 static inline int test_page_attr(struct bitmap *bitmap, int pnum, in test_page_attr() argument
1128 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in test_page_attr()
1131 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum, in test_and_clear_page_attr() argument
1135 bitmap->storage.filemap_attr); in test_and_clear_page_attr()
1139 * to set (and eventually sync) a particular bit in the bitmap file
1144 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_set_bit() argument
1149 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_set_bit()
1150 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_set_bit()
1155 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_set_bit()
1156 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_set_bit()
1158 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
1161 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
1165 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_set_bit()
1172 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_DIRTY); in md_bitmap_file_set_bit()
1175 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_clear_bit() argument
1180 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_clear_bit()
1181 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_clear_bit()
1186 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_clear_bit()
1187 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_clear_bit()
1189 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
1192 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
1194 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_clear_bit()
1199 if (!test_page_attr(bitmap, index - node_offset, BITMAP_PAGE_NEEDWRITE)) { in md_bitmap_file_clear_bit()
1200 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_PENDING); in md_bitmap_file_clear_bit()
1201 bitmap->allclean = 0; in md_bitmap_file_clear_bit()
1205 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_test_bit() argument
1210 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_test_bit()
1213 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
1216 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
1218 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_test_bit()
1228 * sync the dirty pages of the bitmap file to disk */
1229 static void __bitmap_unplug(struct bitmap *bitmap) in __bitmap_unplug() argument
1235 if (!__bitmap_enabled(bitmap)) in __bitmap_unplug()
1240 for (i = 0; i < bitmap->storage.file_pages; i++) { in __bitmap_unplug()
1241 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in __bitmap_unplug()
1242 need_write = test_and_clear_page_attr(bitmap, i, in __bitmap_unplug()
1246 md_bitmap_wait_writes(bitmap); in __bitmap_unplug()
1247 mddev_add_trace_msg(bitmap->mddev, in __bitmap_unplug()
1250 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING); in __bitmap_unplug()
1251 filemap_write_page(bitmap, i, false); in __bitmap_unplug()
1256 md_bitmap_wait_writes(bitmap); in __bitmap_unplug()
1258 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in __bitmap_unplug()
1259 md_bitmap_file_kick(bitmap); in __bitmap_unplug()
1264 struct bitmap *bitmap; member
1273 __bitmap_unplug(unplug_work->bitmap); in md_bitmap_unplug_fn()
1277 static void bitmap_unplug_async(struct bitmap *bitmap) in bitmap_unplug_async() argument
1283 unplug_work.bitmap = bitmap; in bitmap_unplug_async()
1293 struct bitmap *bitmap = mddev->bitmap; in bitmap_unplug() local
1295 if (!bitmap) in bitmap_unplug()
1299 __bitmap_unplug(bitmap); in bitmap_unplug()
1301 bitmap_unplug_async(bitmap); in bitmap_unplug()
1304 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1307 * Initialize the in-memory bitmap from the on-disk bitmap and set up the memory
1308 * mapping of the bitmap file.
1310 * Special case: If there's no bitmap file, or if the bitmap file had been
1315 * This is used when reading an out-of-date bitmap.
1317 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) in md_bitmap_init_from_disk() argument
1319 bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_init_from_disk()
1320 struct mddev *mddev = bitmap->mddev; in md_bitmap_init_from_disk()
1321 unsigned long chunks = bitmap->counts.chunks; in md_bitmap_init_from_disk()
1322 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_init_from_disk()
1330 /* No permanent bitmap - fill with '1s'. */ in md_bitmap_init_from_disk()
1335 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift) in md_bitmap_init_from_disk()
1337 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1338 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1345 pr_warn("%s: bitmap file too short %lu < %lu\n", in md_bitmap_init_from_disk()
1346 bmname(bitmap), in md_bitmap_init_from_disk()
1354 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE)); in md_bitmap_init_from_disk()
1367 ret = read_file_page(file, i, bitmap, count, page); in md_bitmap_init_from_disk()
1376 pr_warn("%s: bitmap file is out of date, doing full recovery\n", in md_bitmap_init_from_disk()
1377 bmname(bitmap)); in md_bitmap_init_from_disk()
1388 * If the bitmap is out of date, dirty the whole page in md_bitmap_init_from_disk()
1395 filemap_write_page(bitmap, i, true); in md_bitmap_init_from_disk()
1396 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) { in md_bitmap_init_from_disk()
1404 struct page *page = filemap_get_page(&bitmap->storage, i); in md_bitmap_init_from_disk()
1405 unsigned long bit = file_page_offset(&bitmap->storage, i); in md_bitmap_init_from_disk()
1410 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_init_from_disk()
1418 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift in md_bitmap_init_from_disk()
1420 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1421 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1427 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n", in md_bitmap_init_from_disk()
1428 bmname(bitmap), store->file_pages, in md_bitmap_init_from_disk()
1434 pr_warn("%s: bitmap initialisation failed: %d\n", in md_bitmap_init_from_disk()
1435 bmname(bitmap), ret); in md_bitmap_init_from_disk()
1439 /* just flag bitmap pages as needing to be written. */
1443 struct bitmap *bitmap = mddev->bitmap; in bitmap_write_all() local
1445 if (!bitmap || !bitmap->storage.filemap) in bitmap_write_all()
1449 if (bitmap->storage.file) in bitmap_write_all()
1452 for (i = 0; i < bitmap->storage.file_pages; i++) in bitmap_write_all()
1453 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE); in bitmap_write_all()
1454 bitmap->allclean = 0; in bitmap_write_all()
1457 static void md_bitmap_count_page(struct bitmap_counts *bitmap, in md_bitmap_count_page() argument
1460 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_count_page()
1462 bitmap->bp[page].count += inc; in md_bitmap_count_page()
1463 md_bitmap_checkfree(bitmap, page); in md_bitmap_count_page()
1466 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset) in md_bitmap_set_pending() argument
1468 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_set_pending()
1470 struct bitmap_page *bp = &bitmap->bp[page]; in md_bitmap_set_pending()
1476 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1499 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1504 struct bitmap *bitmap; in bitmap_daemon_work() local
1514 bitmap = mddev->bitmap; in bitmap_daemon_work()
1515 if (bitmap == NULL) { in bitmap_daemon_work()
1519 if (time_before(jiffies, bitmap->daemon_lastrun in bitmap_daemon_work()
1523 bitmap->daemon_lastrun = jiffies; in bitmap_daemon_work()
1524 if (bitmap->allclean) { in bitmap_daemon_work()
1528 bitmap->allclean = 1; in bitmap_daemon_work()
1530 mddev_add_trace_msg(bitmap->mddev, "md bitmap_daemon_work"); in bitmap_daemon_work()
1536 for (j = 0; j < bitmap->storage.file_pages; j++) in bitmap_daemon_work()
1537 if (test_and_clear_page_attr(bitmap, j, in bitmap_daemon_work()
1539 set_page_attr(bitmap, j, in bitmap_daemon_work()
1542 if (bitmap->need_sync && in bitmap_daemon_work()
1547 bitmap->need_sync = 0; in bitmap_daemon_work()
1548 if (bitmap->storage.filemap) { in bitmap_daemon_work()
1549 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_daemon_work()
1551 cpu_to_le64(bitmap->events_cleared); in bitmap_daemon_work()
1553 set_page_attr(bitmap, 0, in bitmap_daemon_work()
1557 /* Now look at the bitmap counters and if any are '2' or '1', in bitmap_daemon_work()
1560 counts = &bitmap->counts; in bitmap_daemon_work()
1581 if (*bmc == 1 && !bitmap->need_sync) { in bitmap_daemon_work()
1585 md_bitmap_file_clear_bit(bitmap, block); in bitmap_daemon_work()
1589 bitmap->allclean = 0; in bitmap_daemon_work()
1594 md_bitmap_wait_writes(bitmap); in bitmap_daemon_work()
1604 j < bitmap->storage.file_pages in bitmap_daemon_work()
1605 && !test_bit(BITMAP_STALE, &bitmap->flags); in bitmap_daemon_work()
1607 if (test_page_attr(bitmap, j, in bitmap_daemon_work()
1611 if (bitmap->storage.filemap && in bitmap_daemon_work()
1612 test_and_clear_page_attr(bitmap, j, in bitmap_daemon_work()
1614 filemap_write_page(bitmap, j, false); in bitmap_daemon_work()
1618 if (bitmap->allclean == 0) in bitmap_daemon_work()
1623 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap, in md_bitmap_get_counter() argument
1626 __releases(bitmap->lock) in md_bitmap_get_counter()
1627 __acquires(bitmap->lock) in md_bitmap_get_counter()
1633 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_get_counter()
1636 sector_t csize = ((sector_t)1) << bitmap->chunkshift; in md_bitmap_get_counter()
1639 if (page >= bitmap->pages) { in md_bitmap_get_counter()
1648 err = md_bitmap_checkpage(bitmap, page, create, 0); in md_bitmap_get_counter()
1650 if (bitmap->bp[page].hijacked || in md_bitmap_get_counter()
1651 bitmap->bp[page].map == NULL) in md_bitmap_get_counter()
1652 csize = ((sector_t)1) << (bitmap->chunkshift + in md_bitmap_get_counter()
1662 if (bitmap->bp[page].hijacked) { /* hijacked pointer */ in md_bitmap_get_counter()
1667 &bitmap->bp[page].map)[hi]; in md_bitmap_get_counter()
1670 &(bitmap->bp[page].map[pageoff]); in md_bitmap_get_counter()
1676 struct bitmap *bitmap = mddev->bitmap; in bitmap_startwrite() local
1678 if (!bitmap) in bitmap_startwrite()
1685 spin_lock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1686 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1); in bitmap_startwrite()
1688 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1698 prepare_to_wait(&bitmap->overflow_wait, &__wait, in bitmap_startwrite()
1700 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1702 finish_wait(&bitmap->overflow_wait, &__wait); in bitmap_startwrite()
1708 md_bitmap_file_set_bit(bitmap, offset); in bitmap_startwrite()
1709 md_bitmap_count_page(&bitmap->counts, offset, 1); in bitmap_startwrite()
1717 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1731 struct bitmap *bitmap = mddev->bitmap; in bitmap_endwrite() local
1733 if (!bitmap) in bitmap_endwrite()
1741 spin_lock_irqsave(&bitmap->counts.lock, flags); in bitmap_endwrite()
1742 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0); in bitmap_endwrite()
1744 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in bitmap_endwrite()
1748 if (!bitmap->mddev->degraded) { in bitmap_endwrite()
1749 if (bitmap->events_cleared < bitmap->mddev->events) { in bitmap_endwrite()
1750 bitmap->events_cleared = bitmap->mddev->events; in bitmap_endwrite()
1751 bitmap->need_sync = 1; in bitmap_endwrite()
1753 bitmap->sysfs_can_clear); in bitmap_endwrite()
1760 wake_up(&bitmap->overflow_wait); in bitmap_endwrite()
1764 md_bitmap_set_pending(&bitmap->counts, offset); in bitmap_endwrite()
1765 bitmap->allclean = 0; in bitmap_endwrite()
1767 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in bitmap_endwrite()
1776 static bool __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, in __bitmap_start_sync() argument
1782 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */ in __bitmap_start_sync()
1784 return true; /* always resync if no bitmap */ in __bitmap_start_sync()
1786 spin_lock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1789 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in __bitmap_start_sync()
1802 spin_unlock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1822 rv |= __bitmap_start_sync(mddev->bitmap, offset, in bitmap_start_sync()
1831 static void __bitmap_end_sync(struct bitmap *bitmap, sector_t offset, in __bitmap_end_sync() argument
1837 if (bitmap == NULL) { in __bitmap_end_sync()
1841 spin_lock_irqsave(&bitmap->counts.lock, flags); in __bitmap_end_sync()
1842 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in __bitmap_end_sync()
1853 md_bitmap_set_pending(&bitmap->counts, offset); in __bitmap_end_sync()
1854 bitmap->allclean = 0; in __bitmap_end_sync()
1859 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in __bitmap_end_sync()
1865 __bitmap_end_sync(mddev->bitmap, offset, blocks, true); in bitmap_end_sync()
1870 /* Sync has finished, and any bitmap chunks that weren't synced in bitmap_close_sync()
1876 struct bitmap *bitmap = mddev->bitmap; in bitmap_close_sync() local
1878 if (!bitmap) in bitmap_close_sync()
1881 while (sector < bitmap->mddev->resync_max_sectors) { in bitmap_close_sync()
1882 __bitmap_end_sync(bitmap, sector, &blocks, false); in bitmap_close_sync()
1892 struct bitmap *bitmap = mddev->bitmap; in bitmap_cond_end_sync() local
1894 if (!bitmap) in bitmap_cond_end_sync()
1897 bitmap->last_end_sync = jiffies; in bitmap_cond_end_sync()
1900 if (!force && time_before(jiffies, (bitmap->last_end_sync in bitmap_cond_end_sync()
1901 + bitmap->mddev->bitmap_info.daemon_sleep))) in bitmap_cond_end_sync()
1903 wait_event(bitmap->mddev->recovery_wait, in bitmap_cond_end_sync()
1904 atomic_read(&bitmap->mddev->recovery_active) == 0); in bitmap_cond_end_sync()
1906 bitmap->mddev->curr_resync_completed = sector; in bitmap_cond_end_sync()
1907 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags); in bitmap_cond_end_sync()
1908 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1); in bitmap_cond_end_sync()
1910 while (s < sector && s < bitmap->mddev->resync_max_sectors) { in bitmap_cond_end_sync()
1911 __bitmap_end_sync(bitmap, s, &blocks, false); in bitmap_cond_end_sync()
1914 bitmap->last_end_sync = jiffies; in bitmap_cond_end_sync()
1915 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed); in bitmap_cond_end_sync()
1922 struct bitmap *bitmap = mddev->bitmap; in bitmap_sync_with_cluster() local
1926 __bitmap_end_sync(bitmap, sector, &blocks, false); in bitmap_sync_with_cluster()
1938 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed) in md_bitmap_set_memory_bits() argument
1947 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1948 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1); in md_bitmap_set_memory_bits()
1950 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1955 md_bitmap_count_page(&bitmap->counts, offset, 1); in md_bitmap_set_memory_bits()
1956 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_set_memory_bits()
1957 bitmap->allclean = 0; in md_bitmap_set_memory_bits()
1961 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1964 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1969 struct bitmap *bitmap = mddev->bitmap; in bitmap_dirty_bits() local
1971 if (!bitmap) in bitmap_dirty_bits()
1975 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift; in bitmap_dirty_bits()
1977 md_bitmap_set_memory_bits(bitmap, sec, 1); in bitmap_dirty_bits()
1978 md_bitmap_file_set_bit(bitmap, sec); in bitmap_dirty_bits()
1979 if (sec < bitmap->mddev->recovery_cp) in bitmap_dirty_bits()
1984 bitmap->mddev->recovery_cp = sec; in bitmap_dirty_bits()
1990 struct bitmap *bitmap = mddev->bitmap; in bitmap_flush() local
1993 if (!bitmap) /* there was no bitmap */ in bitmap_flush()
2000 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
2002 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
2004 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
2008 bitmap_update_sb(bitmap); in bitmap_flush()
2015 struct bitmap *bitmap = data; in md_bitmap_free() local
2017 if (!bitmap) /* there was no bitmap */ in md_bitmap_free()
2020 if (bitmap->sysfs_can_clear) in md_bitmap_free()
2021 sysfs_put(bitmap->sysfs_can_clear); in md_bitmap_free()
2023 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info && in md_bitmap_free()
2024 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev)) in md_bitmap_free()
2025 md_cluster_stop(bitmap->mddev); in md_bitmap_free()
2028 wait_event(bitmap->write_wait, in md_bitmap_free()
2029 atomic_read(&bitmap->pending_writes) == 0); in md_bitmap_free()
2031 /* release the bitmap file */ in md_bitmap_free()
2032 md_bitmap_file_unmap(&bitmap->storage); in md_bitmap_free()
2034 bp = bitmap->counts.bp; in md_bitmap_free()
2035 pages = bitmap->counts.pages; in md_bitmap_free()
2044 kfree(bitmap); in md_bitmap_free()
2049 struct bitmap *bitmap = mddev->bitmap; in bitmap_start_behind_write() local
2052 if (!bitmap) in bitmap_start_behind_write()
2055 atomic_inc(&bitmap->behind_writes); in bitmap_start_behind_write()
2056 bw = atomic_read(&bitmap->behind_writes); in bitmap_start_behind_write()
2057 if (bw > bitmap->behind_writes_used) in bitmap_start_behind_write()
2058 bitmap->behind_writes_used = bw; in bitmap_start_behind_write()
2061 bw, bitmap->mddev->bitmap_info.max_write_behind); in bitmap_start_behind_write()
2066 struct bitmap *bitmap = mddev->bitmap; in bitmap_end_behind_write() local
2068 if (!bitmap) in bitmap_end_behind_write()
2071 if (atomic_dec_and_test(&bitmap->behind_writes)) in bitmap_end_behind_write()
2072 wake_up(&bitmap->behind_wait); in bitmap_end_behind_write()
2074 atomic_read(&bitmap->behind_writes), in bitmap_end_behind_write()
2075 bitmap->mddev->bitmap_info.max_write_behind); in bitmap_end_behind_write()
2080 struct bitmap *bitmap = mddev->bitmap; in bitmap_wait_behind_writes() local
2083 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { in bitmap_wait_behind_writes()
2087 wait_event(bitmap->behind_wait, in bitmap_wait_behind_writes()
2088 atomic_read(&bitmap->behind_writes) == 0); in bitmap_wait_behind_writes()
2094 struct bitmap *bitmap = mddev->bitmap; in bitmap_destroy() local
2096 if (!bitmap) /* there was no bitmap */ in bitmap_destroy()
2105 mddev->bitmap = NULL; /* disconnect from the md device */ in bitmap_destroy()
2110 md_bitmap_free(bitmap); in bitmap_destroy()
2114 * initialize the bitmap structure
2116 * once mddev->bitmap is set
2118 static struct bitmap *__bitmap_create(struct mddev *mddev, int slot) in __bitmap_create()
2120 struct bitmap *bitmap; in __bitmap_create() local
2131 pr_notice("md/raid:%s: array with journal cannot have bitmap\n", in __bitmap_create()
2136 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); in __bitmap_create()
2137 if (!bitmap) in __bitmap_create()
2140 spin_lock_init(&bitmap->counts.lock); in __bitmap_create()
2141 atomic_set(&bitmap->pending_writes, 0); in __bitmap_create()
2142 init_waitqueue_head(&bitmap->write_wait); in __bitmap_create()
2143 init_waitqueue_head(&bitmap->overflow_wait); in __bitmap_create()
2144 init_waitqueue_head(&bitmap->behind_wait); in __bitmap_create()
2146 bitmap->mddev = mddev; in __bitmap_create()
2147 bitmap->cluster_slot = slot; in __bitmap_create()
2150 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap"); in __bitmap_create()
2152 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear"); in __bitmap_create()
2155 bitmap->sysfs_can_clear = NULL; in __bitmap_create()
2157 bitmap->storage.file = file; in __bitmap_create()
2166 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */ in __bitmap_create()
2170 * instructing us to create a new on-disk bitmap instance. in __bitmap_create()
2173 err = md_bitmap_new_disk_sb(bitmap); in __bitmap_create()
2175 err = md_bitmap_read_sb(bitmap); in __bitmap_create()
2187 bitmap->daemon_lastrun = jiffies; in __bitmap_create()
2188 err = __bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, in __bitmap_create()
2193 pr_debug("created bitmap (%lu pages) for device %s\n", in __bitmap_create()
2194 bitmap->counts.pages, bmname(bitmap)); in __bitmap_create()
2196 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0; in __bitmap_create()
2200 return bitmap; in __bitmap_create()
2202 md_bitmap_free(bitmap); in __bitmap_create()
2208 struct bitmap *bitmap = __bitmap_create(mddev, slot); in bitmap_create() local
2210 if (IS_ERR(bitmap)) in bitmap_create()
2211 return PTR_ERR(bitmap); in bitmap_create()
2213 mddev->bitmap = bitmap; in bitmap_create()
2222 struct bitmap *bitmap = mddev->bitmap; in bitmap_load() local
2225 if (!bitmap) in bitmap_load()
2234 /* Clear out old bitmap info first: Either there is none, or we in bitmap_load()
2247 || bitmap->events_cleared == mddev->events) in bitmap_load()
2253 err = md_bitmap_init_from_disk(bitmap, start); in bitmap_load()
2258 clear_bit(BITMAP_STALE, &bitmap->flags); in bitmap_load()
2261 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery); in bitmap_load()
2266 bitmap_update_sb(bitmap); in bitmap_load()
2268 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in bitmap_load()
2274 /* caller need to free returned bitmap with md_bitmap_free() */
2278 struct bitmap *bitmap; in bitmap_get_from_slot() local
2280 bitmap = __bitmap_create(mddev, slot); in bitmap_get_from_slot()
2281 if (IS_ERR(bitmap)) { in bitmap_get_from_slot()
2282 rv = PTR_ERR(bitmap); in bitmap_get_from_slot()
2286 rv = md_bitmap_init_from_disk(bitmap, 0); in bitmap_get_from_slot()
2288 md_bitmap_free(bitmap); in bitmap_get_from_slot()
2292 return bitmap; in bitmap_get_from_slot()
2295 /* Loads the bitmap associated with slot and copies the resync information
2296 * to our bitmap
2304 struct bitmap *bitmap; in bitmap_copy_from_slot() local
2306 bitmap = bitmap_get_from_slot(mddev, slot); in bitmap_copy_from_slot()
2307 if (IS_ERR(bitmap)) { in bitmap_copy_from_slot()
2308 pr_err("%s can't get bitmap from slot %d\n", __func__, slot); in bitmap_copy_from_slot()
2312 counts = &bitmap->counts; in bitmap_copy_from_slot()
2315 if (md_bitmap_file_test_bit(bitmap, block)) { in bitmap_copy_from_slot()
2319 md_bitmap_file_clear_bit(bitmap, block); in bitmap_copy_from_slot()
2320 md_bitmap_set_memory_bits(mddev->bitmap, block, 1); in bitmap_copy_from_slot()
2321 md_bitmap_file_set_bit(mddev->bitmap, block); in bitmap_copy_from_slot()
2326 bitmap_update_sb(bitmap); in bitmap_copy_from_slot()
2329 for (i = 0; i < bitmap->storage.file_pages; i++) in bitmap_copy_from_slot()
2330 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING)) in bitmap_copy_from_slot()
2331 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE); in bitmap_copy_from_slot()
2332 __bitmap_unplug(bitmap); in bitmap_copy_from_slot()
2334 __bitmap_unplug(mddev->bitmap); in bitmap_copy_from_slot()
2337 md_bitmap_free(bitmap); in bitmap_copy_from_slot()
2344 struct bitmap *bitmap = data; in bitmap_set_pages() local
2346 bitmap->counts.pages = pages; in bitmap_set_pages()
2353 struct bitmap *bitmap = data; in bitmap_get_stats() local
2356 if (!bitmap) in bitmap_get_stats()
2358 if (!bitmap->mddev->bitmap_info.external && in bitmap_get_stats()
2359 !bitmap->storage.sb_page) in bitmap_get_stats()
2361 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_get_stats()
2365 counts = &bitmap->counts; in bitmap_get_stats()
2369 storage = &bitmap->storage; in bitmap_get_stats()
2373 stats->behind_writes = atomic_read(&bitmap->behind_writes); in bitmap_get_stats()
2374 stats->behind_wait = wq_has_sleeper(&bitmap->behind_wait); in bitmap_get_stats()
2375 stats->events_cleared = bitmap->events_cleared; in bitmap_get_stats()
2379 static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, in __bitmap_resize() argument
2384 * Then quiesce, copy bits, replace bitmap, and re-start in __bitmap_resize()
2386 * This function is called both to set up the initial bitmap in __bitmap_resize()
2387 * and to resize the bitmap while the array is active. in __bitmap_resize()
2402 if (bitmap->storage.file && !init) { in __bitmap_resize()
2403 pr_info("md: cannot resize file-based bitmap\n"); in __bitmap_resize()
2412 long space = bitmap->mddev->bitmap_info.space; in __bitmap_resize()
2418 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8); in __bitmap_resize()
2419 if (!bitmap->mddev->bitmap_info.external) in __bitmap_resize()
2422 bitmap->mddev->bitmap_info.space = space; in __bitmap_resize()
2424 chunkshift = bitmap->counts.chunkshift; in __bitmap_resize()
2431 if (!bitmap->mddev->bitmap_info.external) in __bitmap_resize()
2440 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file) in __bitmap_resize()
2442 !bitmap->mddev->bitmap_info.external, in __bitmap_resize()
2443 mddev_is_clustered(bitmap->mddev) in __bitmap_resize()
2444 ? bitmap->cluster_slot : 0); in __bitmap_resize()
2460 bitmap->mddev->pers->quiesce(bitmap->mddev, 1); in __bitmap_resize()
2462 store.file = bitmap->storage.file; in __bitmap_resize()
2463 bitmap->storage.file = NULL; in __bitmap_resize()
2465 if (store.sb_page && bitmap->storage.sb_page) in __bitmap_resize()
2467 page_address(bitmap->storage.sb_page), in __bitmap_resize()
2469 spin_lock_irq(&bitmap->counts.lock); in __bitmap_resize()
2470 md_bitmap_file_unmap(&bitmap->storage); in __bitmap_resize()
2471 bitmap->storage = store; in __bitmap_resize()
2473 old_counts = bitmap->counts; in __bitmap_resize()
2474 bitmap->counts.bp = new_bp; in __bitmap_resize()
2475 bitmap->counts.pages = pages; in __bitmap_resize()
2476 bitmap->counts.missing_pages = pages; in __bitmap_resize()
2477 bitmap->counts.chunkshift = chunkshift; in __bitmap_resize()
2478 bitmap->counts.chunks = chunks; in __bitmap_resize()
2479 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift + in __bitmap_resize()
2485 /* For cluster raid, need to pre-allocate bitmap */ in __bitmap_resize()
2486 if (mddev_is_clustered(bitmap->mddev)) { in __bitmap_resize()
2489 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1); in __bitmap_resize()
2500 bitmap->counts.bp = old_counts.bp; in __bitmap_resize()
2501 bitmap->counts.pages = old_counts.pages; in __bitmap_resize()
2502 bitmap->counts.missing_pages = old_counts.pages; in __bitmap_resize()
2503 bitmap->counts.chunkshift = old_counts.chunkshift; in __bitmap_resize()
2504 bitmap->counts.chunks = old_counts.chunks; in __bitmap_resize()
2505 bitmap->mddev->bitmap_info.chunksize = in __bitmap_resize()
2508 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n"); in __bitmap_resize()
2511 bitmap->counts.bp[page].count += 1; in __bitmap_resize()
2523 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in __bitmap_resize()
2532 md_bitmap_file_set_bit(bitmap, block); in __bitmap_resize()
2536 md_bitmap_count_page(&bitmap->counts, block, 1); in __bitmap_resize()
2537 md_bitmap_set_pending(&bitmap->counts, block); in __bitmap_resize()
2547 if (bitmap->counts.bp != old_counts.bp) { in __bitmap_resize()
2559 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in __bitmap_resize()
2566 md_bitmap_count_page(&bitmap->counts, block, 1); in __bitmap_resize()
2567 md_bitmap_set_pending(&bitmap->counts, block); in __bitmap_resize()
2572 for (i = 0; i < bitmap->storage.file_pages; i++) in __bitmap_resize()
2573 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in __bitmap_resize()
2575 spin_unlock_irq(&bitmap->counts.lock); in __bitmap_resize()
2578 __bitmap_unplug(bitmap); in __bitmap_resize()
2579 bitmap->mddev->pers->quiesce(bitmap->mddev, 0); in __bitmap_resize()
2589 struct bitmap *bitmap = mddev->bitmap; in bitmap_resize() local
2591 if (!bitmap) in bitmap_resize()
2594 return __bitmap_resize(bitmap, blocks, chunksize, init); in bitmap_resize()
2627 if (mddev->bitmap || mddev->bitmap_info.file || in location_store()
2629 /* bitmap already configured. Only option is to clear it */ in location_store()
2643 /* No bitmap, OK to set a location */ in location_store()
2684 /* Ensure new bitmap info is stored in in location_store()
2701 /* 'bitmap/space' is the space available at 'location' for the
2702 * bitmap. This allows the kernel to know when it is safe to
2703 * resize the bitmap to match a resized array.
2714 struct bitmap *bitmap; in space_store() local
2725 bitmap = mddev->bitmap; in space_store()
2726 if (bitmap && sectors < (bitmap->storage.bytes + 511) >> 9) in space_store()
2727 return -EFBIG; /* Bitmap is too big for this small space */ in space_store()
2834 bitmap_update_sb(mddev->bitmap); in backlog_store()
2852 /* Can only be changed when no bitmap is active */ in chunksize_store()
2855 if (mddev->bitmap) in chunksize_store()
2883 if (mddev->bitmap || in metadata_store()
2903 struct bitmap *bitmap; in can_clear_show() local
2906 bitmap = mddev->bitmap; in can_clear_show()
2907 if (bitmap) in can_clear_show()
2908 len = sprintf(page, "%s\n", (bitmap->need_sync ? "false" : in can_clear_show()
2918 struct bitmap *bitmap = mddev->bitmap; in can_clear_store() local
2920 if (!bitmap) in can_clear_store()
2924 bitmap->need_sync = 1; in can_clear_store()
2931 bitmap->need_sync = 0; in can_clear_store()
2945 struct bitmap *bitmap; in behind_writes_used_show() local
2948 bitmap = mddev->bitmap; in behind_writes_used_show()
2949 if (!bitmap) in behind_writes_used_show()
2952 ret = sprintf(page, "%lu\n", bitmap->behind_writes_used); in behind_writes_used_show()
2961 struct bitmap *bitmap = mddev->bitmap; in behind_writes_used_reset() local
2963 if (bitmap) in behind_writes_used_reset()
2964 bitmap->behind_writes_used = 0; in behind_writes_used_reset()
2984 .name = "bitmap",