6  *  Swap reorganised 29.12.95, Stephen Tweedie
17 #include <linux/swap.h>
51 #include "swap.h"
70  * Some modules use swappable objects and may try to swap them out under
72  * check to see if any swap space is available.
83 static const char Bad_file[] = "Bad swap file entry ";
84 static const char Unused_file[] = "Unused swap file entry ";
85 static const char Bad_offset[] = "Bad swap offset entry ";
86 static const char Unused_offset[] = "Unused swap offset entry ";
134  * if one swap device is on the available plist, so the atomic can
152 /* Reclaim the swap entry anyway if possible */
155  * Reclaim the swap entry if there are no more mappings of the
159 /* Reclaim the swap entry if swap is getting full */
199  * returns number of pages in the folio that backs the swap entry. If positive,
201  * folio was associated with the swap entry.
231 	entry = folio->swap;  in __try_to_reclaim_swap()
241 	 * It's safe to delete the folio from swap cache only if the folio's  in __try_to_reclaim_swap()
289  * swapon tell device that all the old swap contents can be discarded,
290  * to allow the swap device to optimize its wear-levelling.
299 	/* Do not discard the swap header page! */  in discard_swap()
347 	struct swap_info_struct *sis = swp_swap_info(folio->swap);  in swap_folio_sector()
352 	offset = swp_offset(folio->swap);  in swap_folio_sector()
359  * swap allocation tell device that a cluster of swap can now be discarded,
360  * to allow the swap device to optimize its wear-levelling.
486 	 * taken by scan_swap_map_slots(), mark the swap entries bad (occupied).  in swap_cluster_schedule_discard()
613 	 * If the swap is discardable, prepare discard the cluster  in free_cluster()
862 		/* in case no swap cache is reclaimed */  in swap_reclaim_full_clusters()
882  * Try to get swap entries with specified order from current cpu's swap entry
897 		/* Serialize HDD SWAP allocation for each device. */  in cluster_alloc_swap_entry()
950 			 * reclaimable (eg. lazy-freed swap cache) slots.  in cluster_alloc_swap_entry()
1187 	 * We try to cluster swap pages by allocating them sequentially  in scan_swap_map_slots()
1188 	 * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this  in scan_swap_map_slots()
1190 	 * a new cluster.  This prevents us from scattering swap pages  in scan_swap_map_slots()
1191 	 * all over the entire swap partition, so that we reduce  in scan_swap_map_slots()
1192 	 * overall disk seek times between swap pages.  -- sct  in scan_swap_map_slots()
1194 	 * And we let swap pages go all over an SSD partition.  Hugh  in scan_swap_map_slots()
1199 		 * page swap is disabled.  Warn and fail the allocation.  in scan_swap_map_slots()
1370  * When we get a swap entry, if there aren't some other ways to
1371  * prevent swapoff, such as the folio in swap cache is locked, RCU
1372  * reader side is locked, etc., the swap entry may become invalid
1373  * because of swapoff.  Then, we need to enclose all swap related
1375  * swap functions call get/put_swap_device() by themselves.
1381  * Check whether swap entry is valid in the swap device.  If so,
1382  * return pointer to swap_info_struct, and keep the swap entry valid
1383  * via preventing the swap device from being swapoff, until
1402  * changing partly because the specified swap entry may be for another
1403  * swap device which has been swapoff.  And in do_swap_page(), after
1404  * the page is read from the swap device, the PTE is verified not
1405  * changed with the page table locked to check whether the swap device
1494  * Drop the last HAS_CACHE flag of swap entries, caller have to
1541  * Caller has made sure that the swap device corresponding to entry
1563  * Called after dropping swapcache to decrease refcnt to swap entries.
1617  * This does not give an exact answer when swap count is continued,
1708 	swp_entry_t entry = folio->swap;  in folio_swapped()
1733 	 * hibernation is allocating its own swap pages for the image,  in folio_swapcache_freeable()
1735 	 * the swap from a folio which has already been recorded in the  in folio_swapcache_freeable()
1736 	 * image as a clean swapcache folio, and then reuse its swap for  in folio_swapcache_freeable()
1739 	 * later read back in from swap, now with the wrong data.  in folio_swapcache_freeable()
1751  * folio_free_swap() - Free the swap space used for this folio.
1754  * If swap is getting full, or if there are no more mappings of this folio,
1755  * then call folio_free_swap to free its swap space.
1757  * Return: true if we were able to release the swap space.
1772  * free_swap_and_cache_nr() - Release reference on range of swap entries and
1777  * For each swap entry in the contiguous range, release a reference. If any swap
1812 	 * Now go back over the range trying to reclaim the swap cache. This is  in free_swap_and_cache_nr()
1814 	 * the swap once per folio in the common case. If we do  in free_swap_and_cache_nr()
1817 	 * page but will only succeed once the swap slot for every subpage is  in free_swap_and_cache_nr()
1824 			 * Folios are always naturally aligned in swap so  in free_swap_and_cache_nr()
1826 			 * folio was found for the swap entry, so advance by 1  in free_swap_and_cache_nr()
1855 	/* This is called for allocating swap entry, not cache */  in get_swap_page_of_type()
1866  * Find the swap type that corresponds to given device (if any).
1869  * from 0, in which the swap header is expected to be located.
1920  * corresponding to given index in swap_info (swap type).
1934  * Return either the total number of swap pages of given type, or the number
1966  * No need to decide whether this PTE shares the swap entry with others,
2018 	 * when reading from swap. This metadata may be indexed by swap entry  in unuse_pte()
2338 		 * swap cache just before we acquired the page lock. The folio  in try_to_unuse()
2339 		 * might even be back in swap cache on another swap area. But  in try_to_unuse()
2350 	 * Lets check again to see if there are still swap entries in the map.  in try_to_unuse()
2352 	 * Under global memory pressure, swap entries can be reinserted back  in try_to_unuse()
2356 	 * above fails, that mm is likely to be freeing swap from  in try_to_unuse()
2359 	 * folio_alloc_swap(), temporarily hiding that swap.  It's easy  in try_to_unuse()
2378  * After a successful try_to_unuse, if no swap is now in use, we know
2468  * A `swap extent' is a simple thing which maps a contiguous range of pages
2469  * onto a contiguous range of disk blocks.  A rbtree of swap extents is
2475  * swap files identically.
2477  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
2487  * For all swap devices we set S_SWAPFILE across the life of the swapon.  This
2488  * prevents users from writing to the swap device, which will corrupt memory.
2490  * The amount of disk space which a single swap extent represents varies.
2548 	 * low-to-high, while swap ordering is high-to-low  in setup_swap_info()
2579 	 * which allocates swap pages from the highest available priority  in _enable_swap_info()
2599 	 * Finished initializing swap device, now it's safe to reference it.  in enable_swap_info()
2731 		/* re-insert swap space back into swap_list */  in SYSCALL_DEFINE1()
2740 	 * Wait for swap operations protected by get/put_swap_device()  in SYSCALL_DEFINE1()
2741 	 * to complete.  Because of synchronize_rcu() here, all swap  in SYSCALL_DEFINE1()
2744 	 * prevent folio_test_swapcache() and the following swap cache  in SYSCALL_DEFINE1()
2787 	/* Destroy swap account information */  in SYSCALL_DEFINE1()
2834 static void *swap_start(struct seq_file *swap, loff_t *pos)  in swap_start()  argument
2855 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)  in swap_next()  argument
2875 static void swap_stop(struct seq_file *swap, void *v)  in swap_stop()  argument
2880 static int swap_show(struct seq_file *swap, void *v)  in swap_show()  argument
2888 		seq_puts(swap, "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n");  in swap_show()
2896 	len = seq_file_path(swap, file, " \t\n\\");  in swap_show()
2897 	seq_printf(swap, "%*s%s\t%lu\t%s%lu\t%s%d\n",  in swap_show()
3037  * Find out how many pages are allowed for a single swap device. There
3039  * 1) the number of bits for the swap offset in the swp_entry_t type, and
3040  * 2) the number of bits in the swap pte, as defined by the different
3043  * In order to find the largest possible bit mask, a swap entry with
3044  * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
3045  * decoded to a swp_entry_t again, and finally the swap offset is
3050  * of a swap pte.
3074 		pr_err("Unable to find swap-space signature\n");  in read_swap_header()
3078 	/* swap partition endianness hack... */  in read_swap_header()
3088 	/* Check the swap header's sub-version */  in read_swap_header()
3090 		pr_warn("Unable to handle swap header version %d\n",  in read_swap_header()
3098 		pr_warn("Empty swap-file\n");  in read_swap_header()
3102 		pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",  in read_swap_header()
3116 		pr_warn("Swap area shorter than signature indicates\n");  in read_swap_header()
3166 		pr_warn("Empty swap-file\n");  in setup_swap_map_and_extents()
3335 	 * Read the swap header.  in SYSCALL_DEFINE2()
3354 	/* OK, set up the swap map and apply the bad block list */  in SYSCALL_DEFINE2()
3374 	 * be above MAX_PAGE_ORDER incase of a large swap file.  in SYSCALL_DEFINE2()
3406 		 * When discard is enabled for swap with no particular  in SYSCALL_DEFINE2()
3407 		 * policy flagged, we set all swap discard flags here in  in SYSCALL_DEFINE2()
3417 		 * perform discards for released swap page-clusters.  in SYSCALL_DEFINE2()
3444 	 * swap device.  in SYSCALL_DEFINE2()
3460 	pr_info("Adding %uk swap on %s.  Priority:%d extents:%d across:%lluk %s%s%s%s\n",  in SYSCALL_DEFINE2()
3529  * Verify that nr swap entries are valid and increment their swap map counts.
3535  * - swap-cache reference is requested but there is already one. -> EEXIST
3536  * - swap-cache reference is requested but the entry is not used. -> ENOENT
3537  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
3564 		 * swapin_readahead() doesn't check if a swap entry is valid, so the  in __swap_duplicate()
3565 		 * swap entry could be SWAP_MAP_BAD. Check here with lock held.  in __swap_duplicate()
3617  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
3626  * Increase reference count of swap entry by 1.
3642  * @entry: first swap entry from which we allocate nr swap cache.
3644  * Called when allocating swap cache for existing swap entries,
3646  * -EEXIST means there is a swap cache.
3671 	return swp_swap_info(folio->swap)->swap_file->f_mapping;  in swapcache_mapping()
3677 	return swap_cache_index(folio->swap);  in __folio_swap_cache_index()
3682  * add_swap_count_continuation - called when a swap count is duplicated
3685  * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
3717 		 * __swap_duplicate(): the swap device may be swapoff  in add_swap_count_continuation()
3730 		 * The higher the swap count, the more likely it is that tasks  in add_swap_count_continuation()
3731 		 * will race to add swap count continuation: we need to avoid  in add_swap_count_continuation()
3798  * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster
3919 	 * We've already scheduled a throttle, avoid taking the global swap  in __folio_throttle_swaprate()
3944 		pr_emerg("Not enough memory for swap heads, swap is disabled\n");  in swapfile_init()