256 	u64 now, state_start;  in get_recent_times()  local
266 		now = cpu_clock(cpu);  in get_recent_times()
287 			times[s] += now - state_start;  in get_recent_times()
396 static void window_reset(struct psi_window *win, u64 now, u64 value,  in window_reset()  argument
399 	win->start_time = now;  in window_reset()
415 static u64 window_update(struct psi_window *win, u64 now, u64 value)  in window_update()  argument
420 	elapsed = now - win->start_time;  in window_update()
430 		window_reset(win, now, value, growth);  in window_update()
441 static void update_triggers(struct psi_group *group, u64 now,  in update_triggers()  argument
478 			growth = window_update(&t->win, now, total[t->state]);  in update_triggers()
487 		if (now < t->last_event_time + t->win.size)  in update_triggers()
497 		t->last_event_time = now;  in update_triggers()
503 static u64 update_averages(struct psi_group *group, u64 now)  in update_averages()  argument
512 	if (now - expires >= psi_period)  in update_averages()
513 		missed_periods = div_u64(now - expires, psi_period);  in update_averages()
523 	period = now - (group->avg_last_update + (missed_periods * psi_period));  in update_averages()
524 	group->avg_last_update = now;  in update_averages()
561 	u64 now;  in psi_avgs_work()  local
568 	now = sched_clock();  in psi_avgs_work()
578 	if (now >= group->avg_next_update) {  in psi_avgs_work()
579 		update_triggers(group, now, PSI_AVGS);  in psi_avgs_work()
580 		group->avg_next_update = update_averages(group, now);  in psi_avgs_work()
585 				group->avg_next_update - now) + 1);  in psi_avgs_work()
591 static void init_rtpoll_triggers(struct psi_group *group, u64 now)  in init_rtpoll_triggers()  argument
596 		window_reset(&t->win, now,  in init_rtpoll_triggers()
600 	group->rtpoll_next_update = now + group->rtpoll_min_period;  in init_rtpoll_triggers()
635 	u64 now;  in psi_rtpoll_work()  local
639 	now = sched_clock();  in psi_rtpoll_work()
641 	if (now > group->rtpoll_until) {  in psi_rtpoll_work()
682 		if (now > group->rtpoll_until)  in psi_rtpoll_work()
683 			init_rtpoll_triggers(group, now);  in psi_rtpoll_work()
690 		group->rtpoll_until = now +  in psi_rtpoll_work()
694 	if (now > group->rtpoll_until) {  in psi_rtpoll_work()
699 	if (now >= group->rtpoll_next_update) {  in psi_rtpoll_work()
701 			update_triggers(group, now, PSI_POLL);  in psi_rtpoll_work()
705 		group->rtpoll_next_update = now + group->rtpoll_min_period;  in psi_rtpoll_work()
709 		nsecs_to_jiffies(group->rtpoll_next_update - now) + 1,  in psi_rtpoll_work()
742 static void record_times(struct psi_group_cpu *groupc, u64 now)  in record_times()  argument
746 	delta = now - groupc->state_start;  in record_times()
747 	groupc->state_start = now;  in record_times()
778 	u64 now;  in psi_group_change()  local
793 	now = cpu_clock(cpu);  in psi_group_change()
842 			record_times(groupc, now);  in psi_group_change()
863 	record_times(groupc, now);  in psi_group_change()
1019 		u64 now;  in psi_account_irqtime()  local
1027 		now = cpu_clock(cpu);  in psi_account_irqtime()
1029 		record_times(groupc, now);  in psi_account_irqtime()
1129 	/* All triggers must be removed by now */  in psi_cgroup_free()
1217 	 * and restart groupc->state_start from now, use .clear = .set = 0  in psi_cgroup_restart()
1238 	u64 now;  in psi_show()  local
1250 	now = sched_clock();  in psi_show()
1252 	if (now >= group->avg_next_update)  in psi_show()
1253 		group->avg_next_update = update_averages(group, now);  in psi_show()