15  * interleave     Allocate memory interleaved over a set of nodes,
23  *                Allocate memory interleaved over a set of nodes based on
29  * bind           Only allocate memory on a specific set of nodes,
33  *                the allocation to memory nodes instead
41  * preferred many Try a set of nodes first before normal fallback. This is
219 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
220 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
236 static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_nodemask()  argument
238 	if (nodes_empty(*nodes))  in mpol_new_nodemask()
240 	pol->nodes = *nodes;  in mpol_new_nodemask()
244 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_preferred()  argument
246 	if (nodes_empty(*nodes))  in mpol_new_preferred()
249 	nodes_clear(pol->nodes);  in mpol_new_preferred()
250 	node_set(first_node(*nodes), pol->nodes);  in mpol_new_preferred()
256  * any, for the new policy.  mpol_new() has already validated the nodes
263 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)  in mpol_set_nodemask()  argument
279 	VM_BUG_ON(!nodes);  in mpol_set_nodemask()
282 		mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);  in mpol_set_nodemask()
284 		nodes_and(nsc->mask2, *nodes, nsc->mask1);  in mpol_set_nodemask()
287 		pol->w.user_nodemask = *nodes;  in mpol_set_nodemask()
297  * initialization. You must invoke mpol_set_nodemask() to set nodes.
300 				  nodemask_t *nodes)  in mpol_new()  argument
305 		if (nodes && !nodes_empty(*nodes))  in mpol_new()
309 	VM_BUG_ON(!nodes);  in mpol_new()
317 		if (nodes_empty(*nodes)) {  in mpol_new()
325 		if (!nodes_empty(*nodes) ||  in mpol_new()
329 	} else if (nodes_empty(*nodes))  in mpol_new()
351 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_default()  argument
355 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_nodemask()  argument
360 		nodes_and(tmp, pol->w.user_nodemask, *nodes);  in mpol_rebind_nodemask()
362 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_nodemask()
364 		nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,  in mpol_rebind_nodemask()
365 								*nodes);  in mpol_rebind_nodemask()
366 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_nodemask()
370 		tmp = *nodes;  in mpol_rebind_nodemask()
372 	pol->nodes = tmp;  in mpol_rebind_nodemask()
376 						const nodemask_t *nodes)  in mpol_rebind_preferred()  argument
378 	pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_preferred()
382  * mpol_rebind_policy - Migrate a policy to a different set of nodes
723 	 * Check page nodes, and queue pages to move, in the current vma.  in queue_pages_test_walk()
748  * If pages found in a given range are not on the required set of @nodes,
761 		nodemask_t *nodes, unsigned long flags,  in queue_pages_range()  argument
768 		.nmask = nodes,  in queue_pages_range()
848 			     nodemask_t *nodes)  in do_set_mempolicy()  argument
857 	new = mpol_new(mode, flags, nodes);  in do_set_mempolicy()
864 	ret = mpol_set_nodemask(new, nodes, scratch);  in do_set_mempolicy()
891 static void get_policy_nodemask(struct mempolicy *pol, nodemask_t *nodes)  in get_policy_nodemask()  argument
893 	nodes_clear(*nodes);  in get_policy_nodemask()
903 		*nodes = pol->nodes;  in get_policy_nodemask()
986 			*policy = next_node_in(current->il_prev, pol->nodes);  in do_get_mempolicy()
993 						       pol->nodes);  in do_get_mempolicy()
1140 	 * This lets us pick a pair of nodes to migrate between, such that  in do_migrate_pages()
1169 			 * However if the number of source nodes is not equal to  in do_migrate_pages()
1170 			 * the number of destination nodes we can not preserve  in do_migrate_pages()
1190 			/* dest not in remaining from nodes? */  in do_migrate_pages()
1357 		 * targeted nodes, for the first VMA to be migrated; for later  in do_mbind()
1358 		 * VMAs, the nodes will still be interleaved from the targeted  in do_mbind()
1438 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,  in get_nodes()  argument
1442 	nodes_clear(*nodes);  in get_nodes()
1449 	 * When the user specified more nodes than supported just check  in get_nodes()
1470 	return get_bitmap(nodes_addr(*nodes), nmask, maxnode);  in get_nodes()
1475 			      nodemask_t *nodes)  in copy_nodes_to_user()  argument
1495 					 nodes_addr(*nodes), maxnode);  in copy_nodes_to_user()
1497 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;  in copy_nodes_to_user()
1524 	nodemask_t nodes;  in kernel_mbind()  local
1533 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_mbind()
1537 	return do_mbind(start, len, lmode, mode_flags, &nodes, flags);  in kernel_mbind()
1619 	nodemask_t nodes;  in kernel_set_mempolicy()  local
1627 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_set_mempolicy()
1631 	return do_set_mempolicy(lmode, mode_flags, &nodes);  in kernel_set_mempolicy()
1690 	/* Is the user allowed to access the target nodes? */  in kernel_migrate_pages()
1743 	nodemask_t nodes;  in kernel_get_mempolicy()  local
1750 	err = do_get_mempolicy(&pval, &nodes, addr, flags);  in kernel_get_mempolicy()
1759 		err = copy_nodes_to_user(nmask, maxnode, &nodes);  in kernel_get_mempolicy()
1868 	 * if policy->nodes has movable memory only,  in apply_policy_zone()
1871 	 * policy->nodes is intersect with node_states[N_MEMORY].  in apply_policy_zone()
1873 	 * policy->nodes has movable memory only.  in apply_policy_zone()
1875 	if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))  in apply_policy_zone()
1890 	if (!current->il_weight || !node_isset(node, policy->nodes)) {  in weighted_interleave_nodes()
1891 		node = next_node_in(node, policy->nodes);  in weighted_interleave_nodes()
1912 		nid = next_node_in(current->il_prev, policy->nodes);  in interleave_nodes()
1938 		return first_node(policy->nodes);  in mempolicy_slab_node()
1959 							&policy->nodes);  in mempolicy_slab_node()
1979 	memcpy(mask, &pol->nodes, sizeof(nodemask_t));  in read_once_policy_nodemask()
2025  * node in pol->nodes (starting from ilx=0), wrapping around if ilx
2026  * exceeds the number of present nodes.
2046  * Return a nodemask representing a mempolicy for filtering nodes for
2057 		*nid = first_node(pol->nodes);  in policy_nodemask()
2060 		nodemask = &pol->nodes;  in policy_nodemask()
2067 		    cpuset_nodemask_valid_mems_allowed(&pol->nodes))  in policy_nodemask()
2068 			nodemask = &pol->nodes;  in policy_nodemask()
2150 		*mask = mempolicy->nodes;  in init_nodemask_of_mempolicy()
2172  * memory allocated from all nodes in system.
2188 		ret = nodes_intersects(mempolicy->nodes, *mask);  in mempolicy_in_oom_domain()
2202 	 * preferred nodes but skip the direct reclaim and allow the  in alloc_pages_preferred_many()
2204 	 * nodes in system.  in alloc_pages_preferred_many()
2243 		 * node and don't fall back to other nodes, as the cost of  in alloc_pages_mpol()
2379 	int nodes;  in alloc_pages_bulk_interleave()  local
2386 	nodes = nodes_weight(pol->nodes);  in alloc_pages_bulk_interleave()
2387 	nr_pages_per_node = nr_pages / nodes;  in alloc_pages_bulk_interleave()
2388 	delta = nr_pages - nodes * nr_pages_per_node;  in alloc_pages_bulk_interleave()
2390 	for (i = 0; i < nodes; i++) {  in alloc_pages_bulk_interleave()
2423 	nodemask_t nodes;  in alloc_pages_bulk_weighted_interleave()  local
2433 	/* read the nodes onto the stack, retry if done during rebind */  in alloc_pages_bulk_weighted_interleave()
2436 		nnodes = read_once_policy_nodemask(pol, &nodes);  in alloc_pages_bulk_weighted_interleave()
2446 	if (weight && node_isset(node, nodes)) {  in alloc_pages_bulk_weighted_interleave()
2476 	for_each_node_mask(node, nodes) {  in alloc_pages_bulk_weighted_interleave()
2491 	resume_node = next_node_in(prev_node, nodes);  in alloc_pages_bulk_weighted_interleave()
2494 		node = next_node_in(prev_node, nodes);  in alloc_pages_bulk_weighted_interleave()
2535 	nr_allocated  = alloc_pages_bulk_noprof(preferred_gfp, nid, &pol->nodes,  in alloc_pages_bulk_preferred_many()
2645 		return !!nodes_equal(a->nodes, b->nodes);  in __mpol_equal()
2792 		if (node_isset(curnid, pol->nodes))  in mpol_misplaced()
2794 		polnid = first_node(pol->nodes);  in mpol_misplaced()
2805 		 * policy nodemask we don't allow numa migration to nodes  in mpol_misplaced()
2814 			 * Optimize placement among multiple nodes  in mpol_misplaced()
2817 			if (node_isset(thisnid, pol->nodes))  in mpol_misplaced()
2825 		 * If no allowed nodes, use current [!misplaced].  in mpol_misplaced()
2827 		if (node_isset(curnid, pol->nodes))  in mpol_misplaced()
2832 				&pol->nodes);  in mpol_misplaced()
3127 			.nodes = nodemask_of_node(nid),  in numa_policy_init()
3133 	 * enabled across suitably sized nodes (default is >= 16MB), or  in numa_policy_init()
3196 	nodemask_t nodes;  in mpol_parse_str()  local
3207 		if (nodelist_parse(nodelist, nodes))  in mpol_parse_str()
3209 		if (!nodes_subset(nodes, node_states[N_MEMORY]))  in mpol_parse_str()
3212 		nodes_clear(nodes);  in mpol_parse_str()
3222 		 * we use first_node(nodes) to grab a single node, so here  in mpol_parse_str()
3223 		 * nodelist (or nodes) cannot be empty.  in mpol_parse_str()
3231 			if (nodes_empty(nodes))  in mpol_parse_str()
3238 		 * Default to online nodes with memory if no nodelist  in mpol_parse_str()
3241 			nodes = node_states[N_MEMORY];  in mpol_parse_str()
3280 	new = mpol_new(mode, mode_flags, &nodes);  in mpol_parse_str()
3285 	 * Save nodes for mpol_to_str() to show the tmpfs mount options  in mpol_parse_str()
3289 		new->nodes = nodes;  in mpol_parse_str()
3291 		nodes_clear(new->nodes);  in mpol_parse_str()
3292 		node_set(first_node(nodes), new->nodes);  in mpol_parse_str()
3298 	 * Save nodes for contextualization: this will be used to "clone"  in mpol_parse_str()
3301 	new->w.user_nodemask = nodes;  in mpol_parse_str()
3331 	nodemask_t nodes = NODE_MASK_NONE;  in mpol_to_str()  local
3352 		nodes = pol->nodes;  in mpol_to_str()
3380 	if (!nodes_empty(nodes))  in mpol_to_str()
3382 			       nodemask_pr_args(&nodes));  in mpol_to_str()