1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10
11 enum hk_flags {
12 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
13 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
14 HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE),
15 };
16
17 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
18 EXPORT_SYMBOL_GPL(housekeeping_overridden);
19
20 struct housekeeping {
21 cpumask_var_t cpumasks[HK_TYPE_MAX];
22 unsigned long flags;
23 };
24
25 static struct housekeeping housekeeping;
26
housekeeping_enabled(enum hk_type type)27 bool housekeeping_enabled(enum hk_type type)
28 {
29 return !!(housekeeping.flags & BIT(type));
30 }
31 EXPORT_SYMBOL_GPL(housekeeping_enabled);
32
housekeeping_any_cpu(enum hk_type type)33 int housekeeping_any_cpu(enum hk_type type)
34 {
35 int cpu;
36
37 if (static_branch_unlikely(&housekeeping_overridden)) {
38 if (housekeeping.flags & BIT(type)) {
39 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
40 if (cpu < nr_cpu_ids)
41 return cpu;
42
43 cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
44 if (likely(cpu < nr_cpu_ids))
45 return cpu;
46 /*
47 * Unless we have another problem this can only happen
48 * at boot time before start_secondary() brings the 1st
49 * housekeeping CPU up.
50 */
51 WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
52 type != HK_TYPE_TIMER);
53 }
54 }
55 return smp_processor_id();
56 }
57 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
58
housekeeping_cpumask(enum hk_type type)59 const struct cpumask *housekeeping_cpumask(enum hk_type type)
60 {
61 if (static_branch_unlikely(&housekeeping_overridden))
62 if (housekeeping.flags & BIT(type))
63 return housekeeping.cpumasks[type];
64 return cpu_possible_mask;
65 }
66 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
67
housekeeping_affine(struct task_struct * t,enum hk_type type)68 void housekeeping_affine(struct task_struct *t, enum hk_type type)
69 {
70 if (static_branch_unlikely(&housekeeping_overridden))
71 if (housekeeping.flags & BIT(type))
72 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
73 }
74 EXPORT_SYMBOL_GPL(housekeeping_affine);
75
housekeeping_test_cpu(int cpu,enum hk_type type)76 bool housekeeping_test_cpu(int cpu, enum hk_type type)
77 {
78 if (static_branch_unlikely(&housekeeping_overridden))
79 if (housekeeping.flags & BIT(type))
80 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
81 return true;
82 }
83 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
84
housekeeping_init(void)85 void __init housekeeping_init(void)
86 {
87 enum hk_type type;
88
89 if (!housekeeping.flags)
90 return;
91
92 static_branch_enable(&housekeeping_overridden);
93
94 if (housekeeping.flags & HK_FLAG_KERNEL_NOISE)
95 sched_tick_offload_init();
96
97 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
98 /* We need at least one CPU to handle housekeeping work */
99 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
100 }
101 }
102
housekeeping_setup_type(enum hk_type type,cpumask_var_t housekeeping_staging)103 static void __init housekeeping_setup_type(enum hk_type type,
104 cpumask_var_t housekeeping_staging)
105 {
106
107 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
108 cpumask_copy(housekeeping.cpumasks[type],
109 housekeeping_staging);
110 }
111
housekeeping_setup(char * str,unsigned long flags)112 static int __init housekeeping_setup(char *str, unsigned long flags)
113 {
114 cpumask_var_t non_housekeeping_mask, housekeeping_staging;
115 unsigned int first_cpu;
116 int err = 0;
117
118 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) {
119 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
120 pr_warn("Housekeeping: nohz unsupported."
121 " Build with CONFIG_NO_HZ_FULL\n");
122 return 0;
123 }
124 }
125
126 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
127 if (cpulist_parse(str, non_housekeeping_mask) < 0) {
128 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
129 goto free_non_housekeeping_mask;
130 }
131
132 alloc_bootmem_cpumask_var(&housekeeping_staging);
133 cpumask_andnot(housekeeping_staging,
134 cpu_possible_mask, non_housekeeping_mask);
135
136 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
137 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
138 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
139 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
140 if (!housekeeping.flags) {
141 pr_warn("Housekeeping: must include one present CPU, "
142 "using boot CPU:%d\n", smp_processor_id());
143 }
144 }
145
146 if (cpumask_empty(non_housekeeping_mask))
147 goto free_housekeeping_staging;
148
149 if (!housekeeping.flags) {
150 /* First setup call ("nohz_full=" or "isolcpus=") */
151 enum hk_type type;
152
153 for_each_set_bit(type, &flags, HK_TYPE_MAX)
154 housekeeping_setup_type(type, housekeeping_staging);
155 } else {
156 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
157 enum hk_type type;
158 unsigned long iter_flags = flags & housekeeping.flags;
159
160 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
161 if (!cpumask_equal(housekeeping_staging,
162 housekeeping.cpumasks[type])) {
163 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
164 goto free_housekeeping_staging;
165 }
166 }
167
168 iter_flags = flags & ~housekeeping.flags;
169
170 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
171 housekeeping_setup_type(type, housekeeping_staging);
172 }
173
174 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE))
175 tick_nohz_full_setup(non_housekeeping_mask);
176
177 housekeeping.flags |= flags;
178 err = 1;
179
180 free_housekeeping_staging:
181 free_bootmem_cpumask_var(housekeeping_staging);
182 free_non_housekeeping_mask:
183 free_bootmem_cpumask_var(non_housekeeping_mask);
184
185 return err;
186 }
187
housekeeping_nohz_full_setup(char * str)188 static int __init housekeeping_nohz_full_setup(char *str)
189 {
190 unsigned long flags;
191
192 flags = HK_FLAG_KERNEL_NOISE;
193
194 return housekeeping_setup(str, flags);
195 }
196 __setup("nohz_full=", housekeeping_nohz_full_setup);
197
housekeeping_isolcpus_setup(char * str)198 static int __init housekeeping_isolcpus_setup(char *str)
199 {
200 unsigned long flags = 0;
201 bool illegal = false;
202 char *par;
203 int len;
204
205 while (isalpha(*str)) {
206 /*
207 * isolcpus=nohz is equivalent to nohz_full.
208 */
209 if (!strncmp(str, "nohz,", 5)) {
210 str += 5;
211 flags |= HK_FLAG_KERNEL_NOISE;
212 continue;
213 }
214
215 if (!strncmp(str, "domain,", 7)) {
216 str += 7;
217 flags |= HK_FLAG_DOMAIN;
218 continue;
219 }
220
221 if (!strncmp(str, "managed_irq,", 12)) {
222 str += 12;
223 flags |= HK_FLAG_MANAGED_IRQ;
224 continue;
225 }
226
227 /*
228 * Skip unknown sub-parameter and validate that it is not
229 * containing an invalid character.
230 */
231 for (par = str, len = 0; *str && *str != ','; str++, len++) {
232 if (!isalpha(*str) && *str != '_')
233 illegal = true;
234 }
235
236 if (illegal) {
237 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
238 return 0;
239 }
240
241 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
242 str++;
243 }
244
245 /* Default behaviour for isolcpus without flags */
246 if (!flags)
247 flags |= HK_FLAG_DOMAIN;
248
249 return housekeeping_setup(str, flags);
250 }
251 __setup("isolcpus=", housekeeping_isolcpus_setup);
252