2  * cpuidle.c - core cpuidle infrastructure
4  * (C) 2006-2007 Venkatesh Pallipadi <[email protected]>
11 #include "linux/percpu-defs.h"
17 #include <linux/sched/idle.h>
20 #include <linux/cpu.h>
55 	return off || !initialized || !drv || !dev || !dev->enabled;  in cpuidle_not_available()
59  * cpuidle_play_dead - cpu off-lining
70 		return -ENODEV;  in cpuidle_play_dead()
72 	for (i = drv->state_count - 1; i >= 0; i--) {  in cpuidle_play_dead()
73 		if (drv->states[i].enter_dead)  in cpuidle_play_dead()
74 			drv->states[i].enter_dead(dev, i);  in cpuidle_play_dead()
81 	return -ENODEV;  in cpuidle_play_dead()
93 	for (i = 1; i < drv->state_count; i++) {  in find_deepest_state()
94 		struct cpuidle_state *s = &drv->states[i];  in find_deepest_state()
96 		if (dev->states_usage[i].disable ||  in find_deepest_state()
97 		    s->exit_latency_ns <= latency_req ||  in find_deepest_state()
98 		    s->exit_latency_ns > max_latency_ns ||  in find_deepest_state()
99 		    (s->flags & forbidden_flags) ||  in find_deepest_state()
100 		    (s2idle && !s->enter_s2idle))  in find_deepest_state()
103 		latency_req = s->exit_latency_ns;  in find_deepest_state()
110  * cpuidle_use_deepest_state - Set/unset governor override mode.
111  * @latency_limit_ns: Idle state exit latency limit (or no override if 0).
113  * If @latency_limit_ns is nonzero, set the current CPU to use the deepest idle
124 		dev->forced_idle_latency_limit_ns = latency_limit_ns;  in cpuidle_use_deepest_state()
129  * cpuidle_find_deepest_state - Find the deepest available idle state.
130  * @drv: cpuidle driver for the given CPU.
131  * @dev: cpuidle device for the given CPU.
132  * @latency_limit_ns: Idle state exit latency limit
134  * Return: the index of the deepest available idle state.
147 	struct cpuidle_state *target_state = &drv->states[index];  in enter_s2idle_proper()
161 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in enter_s2idle_proper()
166 	target_state->enter_s2idle(dev, drv, index);  in enter_s2idle_proper()
169 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in enter_s2idle_proper()
178 	dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);  in enter_s2idle_proper()
179 	dev->states_usage[index].s2idle_usage++;  in enter_s2idle_proper()
184  * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle.
185  * @drv: cpuidle driver for the given CPU.
186  * @dev: cpuidle device for the given CPU.
188  * If there are states with the ->enter_s2idle callback, find the deepest of
196 	 * Find the deepest state with ->enter_s2idle present, which guarantees  in cpuidle_enter_s2idle()
210  * cpuidle_enter_state - enter the state and update stats
211  * @dev: cpuidle device for this cpu
212  * @drv: cpuidle driver for this cpu
213  * @index: index into the states table in @drv of the state to enter
221 	struct cpuidle_state *target_state = &drv->states[index];  in cpuidle_enter_state()
222 	bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);  in cpuidle_enter_state()
230 	 * CPU as a broadcast timer, this call may fail if it is not available.  in cpuidle_enter_state()
233 		index = find_deepest_state(drv, dev, target_state->exit_latency_ns,  in cpuidle_enter_state()
236 		target_state = &drv->states[index];  in cpuidle_enter_state()
240 	if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED)  in cpuidle_enter_state()
243 	/* Take note of the planned idle state. */  in cpuidle_enter_state()
246 	trace_cpu_idle(index, dev->cpu);  in cpuidle_enter_state()
250 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in cpuidle_enter_state()
266 	 * functions called within the RCU-idle region.  in cpuidle_enter_state()
268 	entered_state = target_state->enter(dev, drv, index);  in cpuidle_enter_state()
270 	if (WARN_ONCE(!irqs_disabled(), "%ps leaked IRQ state", target_state->enter))  in cpuidle_enter_state()
273 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in cpuidle_enter_state()
281 	trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);  in cpuidle_enter_state()
283 	/* The cpu is no longer idle or about to enter idle. */  in cpuidle_enter_state()
293 		s64 diff, delay = drv->states[entered_state].exit_latency_ns;  in cpuidle_enter_state()
303 		dev->last_residency_ns = diff;  in cpuidle_enter_state()
304 		dev->states_usage[entered_state].time_ns += diff;  in cpuidle_enter_state()
305 		dev->states_usage[entered_state].usage++;  in cpuidle_enter_state()
307 		if (diff < drv->states[entered_state].target_residency_ns) {  in cpuidle_enter_state()
308 			for (i = entered_state - 1; i >= 0; i--) {  in cpuidle_enter_state()
309 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
312 				/* Shallower states are enabled, so update. */  in cpuidle_enter_state()
313 				dev->states_usage[entered_state].above++;  in cpuidle_enter_state()
314 				trace_cpu_idle_miss(dev->cpu, entered_state, false);  in cpuidle_enter_state()
318 			for (i = entered_state + 1; i < drv->state_count; i++) {  in cpuidle_enter_state()
319 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
324 				 * better match for the observed idle duration.  in cpuidle_enter_state()
326 				if (diff - delay >= drv->states[i].target_residency_ns) {  in cpuidle_enter_state()
327 					dev->states_usage[entered_state].below++;  in cpuidle_enter_state()
328 					trace_cpu_idle_miss(dev->cpu, entered_state, true);  in cpuidle_enter_state()
335 		dev->last_residency_ns = 0;  in cpuidle_enter_state()
336 		dev->states_usage[index].rejected++;  in cpuidle_enter_state()
345  * cpuidle_select - ask the cpuidle framework to choose an idle state
351  * Returns the index of the idle state.  The return value must not be negative.
360 	return cpuidle_curr_governor->select(drv, dev, stop_tick);  in cpuidle_select()
364  * cpuidle_enter - enter into the specified idle state
366  * @drv:   the cpuidle driver tied with the cpu
368  * @index: the index in the idle state table
370  * Returns the index in the idle state, < 0 in case of error.
382 	 * ->select() callback have decided, whether to stop the tick or not.  in cpuidle_enter()
384 	WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());  in cpuidle_enter()
391 	WRITE_ONCE(dev->next_hrtimer, 0);  in cpuidle_enter()
396  * cpuidle_reflect - tell the underlying governor what was the state
400  * @index: the index in the idle state table
405 	if (cpuidle_curr_governor->reflect && index >= 0)  in cpuidle_reflect()
406 		cpuidle_curr_governor->reflect(dev, index);  in cpuidle_reflect()
411  * for most users, the time for a single ping-pong workload like
415  * perf bench sched pipe -l 10000
423  * cpuidle_poll_time - return amount of time to poll for,
424  * governors can override dev->poll_limit_ns if necessary
426  * @drv:   the cpuidle driver tied with the cpu
438 	if (dev->poll_limit_ns)  in cpuidle_poll_time()
439 		return dev->poll_limit_ns;  in cpuidle_poll_time()
442 	for (i = 1; i < drv->state_count; i++) {  in cpuidle_poll_time()
445 		if (dev->states_usage[i].disable)  in cpuidle_poll_time()
448 		state_limit = drv->states[i].target_residency_ns;  in cpuidle_poll_time()
456 	dev->poll_limit_ns = limit_ns;  in cpuidle_poll_time()
458 	return dev->poll_limit_ns;  in cpuidle_poll_time()
462  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
467 		/* Make sure all changes finished before we switch to new idle */  in cpuidle_install_idle_handler()
474  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
485 	 * are done looking at pointed idle states.  in cpuidle_uninstall_idle_handler()
491  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
502  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
529  * cpuidle_enable_device - enables idle PM for a CPU
530  * @dev: the CPU
541 		return -EINVAL;  in cpuidle_enable_device()
543 	if (dev->enabled)  in cpuidle_enable_device()
547 		return -EIO;  in cpuidle_enable_device()
552 		return -EIO;  in cpuidle_enable_device()
554 	if (!dev->registered)  in cpuidle_enable_device()
555 		return -EINVAL;  in cpuidle_enable_device()
561 	if (cpuidle_curr_governor->enable) {  in cpuidle_enable_device()
562 		ret = cpuidle_curr_governor->enable(drv, dev);  in cpuidle_enable_device()
569 	dev->enabled = 1;  in cpuidle_enable_device()
583  * cpuidle_disable_device - disables idle PM for a CPU
584  * @dev: the CPU
593 	if (!dev || !dev->enabled)  in cpuidle_disable_device()
599 	dev->enabled = 0;  in cpuidle_disable_device()
601 	if (cpuidle_curr_governor->disable)  in cpuidle_disable_device()
602 		cpuidle_curr_governor->disable(drv, dev);  in cpuidle_disable_device()
605 	enabled_devices--;  in cpuidle_disable_device()
614 	list_del(&dev->device_list);  in __cpuidle_unregister_device()
615 	per_cpu(cpuidle_devices, dev->cpu) = NULL;  in __cpuidle_unregister_device()
616 	module_put(drv->owner);  in __cpuidle_unregister_device()
618 	dev->registered = 0;  in __cpuidle_unregister_device()
623 	memset(dev->states_usage, 0, sizeof(dev->states_usage));  in __cpuidle_device_init()
624 	dev->last_residency_ns = 0;  in __cpuidle_device_init()
625 	dev->next_hrtimer = 0;  in __cpuidle_device_init()
629  * __cpuidle_register_device - internal register function called before register
631  * @dev: the cpu
640 	if (!try_module_get(drv->owner))  in __cpuidle_register_device()
641 		return -EINVAL;  in __cpuidle_register_device()
643 	for (i = 0; i < drv->state_count; i++) {  in __cpuidle_register_device()
644 		if (drv->states[i].flags & CPUIDLE_FLAG_UNUSABLE)  in __cpuidle_register_device()
645 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;  in __cpuidle_register_device()
647 		if (drv->states[i].flags & CPUIDLE_FLAG_OFF)  in __cpuidle_register_device()
648 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER;  in __cpuidle_register_device()
651 	per_cpu(cpuidle_devices, dev->cpu) = dev;  in __cpuidle_register_device()
652 	list_add(&dev->device_list, &cpuidle_detected_devices);  in __cpuidle_register_device()
658 		dev->registered = 1;  in __cpuidle_register_device()
664  * cpuidle_register_device - registers a CPU's idle PM feature
665  * @dev: the cpu
669 	int ret = -EBUSY;  in cpuidle_register_device()
672 		return -EINVAL;  in cpuidle_register_device()
676 	if (dev->registered)  in cpuidle_register_device()
710  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
711  * @dev: the cpu
715 	if (!dev || dev->registered == 0)  in cpuidle_unregister_device()
742 	int cpu;  in cpuidle_unregister()  local
745 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_unregister()
746 		device = &per_cpu(cpuidle_dev, cpu);  in cpuidle_unregister()
755  * cpuidle_register: registers the driver and the cpu devices with the
761  * @coupled_cpus: a cpumask for the coupled states
768 	int ret, cpu;  in cpuidle_register()  local
777 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_register()
778 		device = &per_cpu(cpuidle_dev, cpu);  in cpuidle_register()
779 		device->cpu = cpu;  in cpuidle_register()
783 		 * On multiplatform for ARM, the coupled idle states could be  in cpuidle_register()
788 			device->coupled_cpus = *coupled_cpus;  in cpuidle_register()
794 		pr_err("Failed to register cpuidle device for cpu%d\n", cpu);  in cpuidle_register()
805  * cpuidle_init - core initializer
810 		return -ENODEV;  in cpuidle_init()