7  * set of CPUs in a system, one bit position per CPU number.  In general,
82  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
83  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
84  *     cpu_enabled_mask  - has bit 'cpu' set iff cpu can be brought online
85  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
86  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
90  *  The cpu_possible_mask is fixed at boot time, as the set of CPU IDs
106  *    assumption that their single CPU is online.  The UP
112  *    only one CPU.
132 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)  in cpu_max_bits_warn()  argument
135 	WARN_ON_ONCE(cpu >= bits);  in cpu_max_bits_warn()
139 /* verify cpu argument to cpumask_* operators */
140 static __always_inline unsigned int cpumask_check(unsigned int cpu)  in cpumask_check()  argument
142 	cpu_max_bits_warn(cpu, small_cpumask_bits);  in cpumask_check()
143 	return cpu;  in cpumask_check()
147  * cpumask_first - get the first cpu in a cpumask
158  * cpumask_first_zero - get the first unset cpu in a cpumask
169  * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
182  * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3
199  * cpumask_last - get the last CPU in a cpumask
210  * cpumask_next - get the next cpu in a cpumask
211  * @n: the cpu prior to the place to search (i.e. return will be > @n)
226  * cpumask_next_zero - get the next unset cpu in a cpumask
227  * @n: the cpu prior to the place to search (i.e. return will be > @n)
242 /* Uniprocessor: there is only one valid CPU */
269  * cpumask_next_and - get the next cpu in *src1p & *src2p
270  * @n: the cpu prior to the place to search (i.e. return will be > @n)
288  * for_each_cpu - iterate over every cpu in a mask
289  * @cpu: the (optionally unsigned) integer iterator
292  * After the loop, cpu is >= nr_cpu_ids.
294 #define for_each_cpu(cpu, mask)				\  argument
295 	for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
306 	 * Return the first available CPU when wrapping, or when starting before cpu0,  in cpumask_next_wrap()
319  * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
320  * @cpu: the (optionally unsigned) integer iterator
326  * After the loop, cpu is >= nr_cpu_ids.
328 #define for_each_cpu_wrap(cpu, mask, start)				\  argument
329 	for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
332  * for_each_cpu_and - iterate over every cpu in both masks
333  * @cpu: the (optionally unsigned) integer iterator
337  * This saves a temporary CPU mask in many places.  It is equivalent to:
340  *	for_each_cpu(cpu, &tmp)
343  * After the loop, cpu is >= nr_cpu_ids.
345 #define for_each_cpu_and(cpu, mask1, mask2)				\  argument
346 	for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
349  * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
351  * @cpu: the (optionally unsigned) integer iterator
355  * This saves a temporary CPU mask in many places.  It is equivalent to:
358  *	for_each_cpu(cpu, &tmp)
361  * After the loop, cpu is >= nr_cpu_ids.
363 #define for_each_cpu_andnot(cpu, mask1, mask2)				\  argument
364 	for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
367  * for_each_cpu_or - iterate over every cpu present in either mask
368  * @cpu: the (optionally unsigned) integer iterator
372  * This saves a temporary CPU mask in many places.  It is equivalent to:
375  *	for_each_cpu(cpu, &tmp)
378  * After the loop, cpu is >= nr_cpu_ids.
380 #define for_each_cpu_or(cpu, mask1, mask2)				\  argument
381 	for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
384  * for_each_cpu_from - iterate over CPUs present in @mask, from @cpu to the end of @mask.
385  * @cpu: the (optionally unsigned) integer iterator
388  * After the loop, cpu is >= nr_cpu_ids.
390 #define for_each_cpu_from(cpu, mask)				\  argument
391 	for_each_set_bit_from(cpu, cpumask_bits(mask), small_cpumask_bits)
394  * cpumask_any_but - return an arbitrary cpu in a cpumask, but not this one.
396  * @cpu: the cpu to ignore.
398  * Often used to find any cpu but smp_processor_id() in a mask.
402 unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)  in cpumask_any_but()  argument
406 	cpumask_check(cpu);  in cpumask_any_but()
408 		if (i != cpu)  in cpumask_any_but()
414  * cpumask_any_and_but - pick an arbitrary cpu from *mask1 & *mask2, but not this one.
417  * @cpu: the cpu to ignore
424 				 unsigned int cpu)  in cpumask_any_and_but()
428 	cpumask_check(cpu);  in cpumask_any_and_but()
430 	if (i != cpu)  in cpumask_any_and_but()
433 	return cpumask_next_and(cpu, mask1, mask2);  in cpumask_any_and_but()
437  * cpumask_nth - get the Nth cpu in a cpumask
439  * @cpu: the Nth cpu to find, starting from 0
441  * Return: >= nr_cpu_ids if such cpu doesn't exist.
444 unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)  in cpumask_nth()  argument
446 	return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth()
450  * cpumask_nth_and - get the Nth cpu in 2 cpumasks
453  * @cpu: the Nth cpu to find, starting from 0
455  * Return: >= nr_cpu_ids if such cpu doesn't exist.
458 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_and()  argument
462 				small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_and()
466  * cpumask_nth_andnot - get the Nth cpu set in 1st cpumask, and clear in 2nd.
469  * @cpu: the Nth cpu to find, starting from 0
471  * Return: >= nr_cpu_ids if such cpu doesn't exist.
474 unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_andnot()  argument
478 				small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_andnot()
482  * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
486  * @cpu: the Nth cpu to find, starting from 0
488  * Return: >= nr_cpu_ids if such cpu doesn't exist.
491 unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_and_andnot()  argument
498 					small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_and_andnot()
512  * cpumask_set_cpu - set a cpu in a cpumask
513  * @cpu: cpu number (< nr_cpu_ids)
517 void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)  in cpumask_set_cpu()  argument
519 	set_bit(cpumask_check(cpu), cpumask_bits(dstp));  in cpumask_set_cpu()
523 void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)  in __cpumask_set_cpu()  argument
525 	__set_bit(cpumask_check(cpu), cpumask_bits(dstp));  in __cpumask_set_cpu()
530  * cpumask_clear_cpu - clear a cpu in a cpumask
531  * @cpu: cpu number (< nr_cpu_ids)
534 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)  in cpumask_clear_cpu()  argument
536 	clear_bit(cpumask_check(cpu), cpumask_bits(dstp));  in cpumask_clear_cpu()
539 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)  in __cpumask_clear_cpu()  argument
541 	__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));  in __cpumask_clear_cpu()
545  * cpumask_assign_cpu - assign a cpu in a cpumask
546  * @cpu: cpu number (< nr_cpu_ids)
550 static __always_inline void cpumask_assign_cpu(int cpu, struct cpumask *dstp, bool value)  in cpumask_assign_cpu()  argument
552 	assign_bit(cpumask_check(cpu), cpumask_bits(dstp), value);  in cpumask_assign_cpu()
555 static __always_inline void __cpumask_assign_cpu(int cpu, struct cpumask *dstp, bool value)  in __cpumask_assign_cpu()  argument
557 	__assign_bit(cpumask_check(cpu), cpumask_bits(dstp), value);  in __cpumask_assign_cpu()
561  * cpumask_test_cpu - test for a cpu in a cpumask
562  * @cpu: cpu number (< nr_cpu_ids)
565  * Return: true if @cpu is set in @cpumask, else returns false
568 bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)  in cpumask_test_cpu()  argument
570 	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));  in cpumask_test_cpu()
574  * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
575  * @cpu: cpu number (< nr_cpu_ids)
580  * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
583 bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)  in cpumask_test_and_set_cpu()  argument
585 	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));  in cpumask_test_and_set_cpu()
589  * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
590  * @cpu: cpu number (< nr_cpu_ids)
595  * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
598 bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)  in cpumask_test_and_clear_cpu()  argument
600 	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));  in cpumask_test_and_clear_cpu()
843  * cpumask_any - pick an arbitrary cpu from *srcp
851  * cpumask_any_and - pick an arbitrary cpu from *mask1 & *mask2
860  * cpumask_of - the cpumask containing just a given cpu
861  * @cpu: the cpu (<= nr_cpu_ids)
863 #define cpumask_of(cpu) (get_cpu_mask(cpu))  argument
1033 #define for_each_possible_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
1034 #define for_each_online_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
1035 #define for_each_present_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
1037 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)  argument
1038 #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)  argument
1039 #define for_each_enabled_cpu(cpu)   for_each_cpu((cpu), cpu_enabled_mask)  argument
1040 #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)  argument
1047 #define assign_cpu(cpu, mask, val)	\  argument
1048 	assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
1050 #define set_cpu_possible(cpu, possible)	assign_cpu((cpu), &__cpu_possible_mask, (possible))  argument
1051 #define set_cpu_enabled(cpu, enabled)	assign_cpu((cpu), &__cpu_enabled_mask, (enabled))  argument
1052 #define set_cpu_present(cpu, present)	assign_cpu((cpu), &__cpu_present_mask, (present))  argument
1053 #define set_cpu_active(cpu, active)	assign_cpu((cpu), &__cpu_active_mask, (active))  argument
1054 #define set_cpu_dying(cpu, dying)	assign_cpu((cpu), &__cpu_dying_mask, (dying))  argument
1056 void set_cpu_online(unsigned int cpu, bool online);
1078  * Special-case data structure for "single bit set only" constant CPU masks.
1087 static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu)  in get_cpu_mask()  argument
1089 	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];  in get_cpu_mask()
1090 	p -= cpu / BITS_PER_LONG;  in get_cpu_mask()
1100  * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1114 static __always_inline bool cpu_online(unsigned int cpu)  in cpu_online()  argument
1116 	return cpumask_test_cpu(cpu, cpu_online_mask);  in cpu_online()
1119 static __always_inline bool cpu_enabled(unsigned int cpu)  in cpu_enabled()  argument
1121 	return cpumask_test_cpu(cpu, cpu_enabled_mask);  in cpu_enabled()
1124 static __always_inline bool cpu_possible(unsigned int cpu)  in cpu_possible()  argument
1126 	return cpumask_test_cpu(cpu, cpu_possible_mask);  in cpu_possible()
1129 static __always_inline bool cpu_present(unsigned int cpu)  in cpu_present()  argument
1131 	return cpumask_test_cpu(cpu, cpu_present_mask);  in cpu_present()
1134 static __always_inline bool cpu_active(unsigned int cpu)  in cpu_active()  argument
1136 	return cpumask_test_cpu(cpu, cpu_active_mask);  in cpu_active()
1139 static __always_inline bool cpu_dying(unsigned int cpu)  in cpu_dying()  argument
1141 	return cpumask_test_cpu(cpu, cpu_dying_mask);  in cpu_dying()
1152 static __always_inline bool cpu_online(unsigned int cpu)  in cpu_online()  argument
1154 	return cpu == 0;  in cpu_online()
1157 static __always_inline bool cpu_possible(unsigned int cpu)  in cpu_possible()  argument
1159 	return cpu == 0;  in cpu_possible()
1162 static __always_inline bool cpu_enabled(unsigned int cpu)  in cpu_enabled()  argument
1164 	return cpu == 0;  in cpu_enabled()
1167 static __always_inline bool cpu_present(unsigned int cpu)  in cpu_present()  argument
1169 	return cpu == 0;  in cpu_present()
1172 static __always_inline bool cpu_active(unsigned int cpu)  in cpu_active()  argument
1174 	return cpu == 0;  in cpu_active()
1177 static __always_inline bool cpu_dying(unsigned int cpu)  in cpu_dying()  argument
1184 #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))  argument
1296  * cover a worst-case of every other cpu being on one of two nodes for a