53 static struct drm_i915_private *rpm_to_i915(struct intel_runtime_pm *rpm)  in rpm_to_i915()  argument
55 	return container_of(rpm, struct drm_i915_private, runtime_pm);  in rpm_to_i915()
60 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
62 	ref_tracker_dir_init(&rpm->debug, INTEL_REFTRACK_DEAD_COUNT, dev_name(rpm->kdev));  in init_intel_runtime_pm_wakeref()
66 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
68 	if (!rpm->available || rpm->no_wakeref_tracking)  in track_intel_runtime_pm_wakeref()
71 	return intel_ref_tracker_alloc(&rpm->debug);  in track_intel_runtime_pm_wakeref()
74 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
77 	if (!rpm->available || rpm->no_wakeref_tracking)  in untrack_intel_runtime_pm_wakeref()
80 	intel_ref_tracker_free(&rpm->debug, wakeref);  in untrack_intel_runtime_pm_wakeref()
83 static void untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
85 	ref_tracker_dir_exit(&rpm->debug);  in untrack_all_intel_runtime_pm_wakerefs()
89 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
93 	if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,  in __intel_wakeref_dec_and_check_tracking()
94 					 &rpm->debug.lock,  in __intel_wakeref_dec_and_check_tracking()
98 	ref_tracker_dir_print_locked(&rpm->debug, INTEL_REFTRACK_PRINT_LIMIT);  in __intel_wakeref_dec_and_check_tracking()
99 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in __intel_wakeref_dec_and_check_tracking()
102 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in print_intel_runtime_pm_wakeref()  argument
105 	intel_ref_tracker_show(&rpm->debug, p);  in print_intel_runtime_pm_wakeref()
110 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
115 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
120 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
126 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
128 	atomic_dec(&rpm->wakeref_count);  in __intel_wakeref_dec_and_check_tracking()
132 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
139 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)  in intel_runtime_pm_acquire()  argument
142 		atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_acquire()
143 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_acquire()
145 		atomic_inc(&rpm->wakeref_count);  in intel_runtime_pm_acquire()
146 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_acquire()
151 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)  in intel_runtime_pm_release()  argument
154 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_release()
155 		atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_release()
157 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_release()
160 	__intel_wakeref_dec_and_check_tracking(rpm);  in intel_runtime_pm_release()
163 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get()  argument
166 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in __intel_runtime_pm_get()
169 	ret = pm_runtime_get_sync(rpm->kdev);  in __intel_runtime_pm_get()
173 	intel_runtime_pm_acquire(rpm, wakelock);  in __intel_runtime_pm_get()
175 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get()
180  * @rpm: the intel_runtime_pm structure
195 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_raw()  argument
197 	return __intel_runtime_pm_get(rpm, false);  in intel_runtime_pm_get_raw()
202  * @rpm: the intel_runtime_pm structure
212 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get()  argument
214 	return __intel_runtime_pm_get(rpm, true);  in intel_runtime_pm_get()
219  * @rpm: the intel_runtime_pm structure
239 static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get_if_active()  argument
244 		 * In cases runtime PM is disabled by the RPM core and we get  in __intel_runtime_pm_get_if_active()
250 		     pm_runtime_get_if_active(rpm->kdev) <= 0) ||  in __intel_runtime_pm_get_if_active()
252 		     pm_runtime_get_if_in_use(rpm->kdev) <= 0))  in __intel_runtime_pm_get_if_active()
256 	intel_runtime_pm_acquire(rpm, true);  in __intel_runtime_pm_get_if_active()
258 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get_if_active()
261 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_in_use()  argument
263 	return __intel_runtime_pm_get_if_active(rpm, false);  in intel_runtime_pm_get_if_in_use()
266 intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_active()  argument
268 	return __intel_runtime_pm_get_if_active(rpm, true);  in intel_runtime_pm_get_if_active()
273  * @rpm: the intel_runtime_pm structure
286 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_noresume()  argument
288 	assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_get_noresume()
289 	pm_runtime_get_noresume(rpm->kdev);  in intel_runtime_pm_get_noresume()
291 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_noresume()
293 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_noresume()
296 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_put()  argument
300 	struct device *kdev = rpm->kdev;  in __intel_runtime_pm_put()
302 	untrack_intel_runtime_pm_wakeref(rpm, wref);  in __intel_runtime_pm_put()
304 	intel_runtime_pm_release(rpm, wakelock);  in __intel_runtime_pm_put()
312  * @rpm: the intel_runtime_pm structure
320 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put_raw()  argument
322 	__intel_runtime_pm_put(rpm, wref, false);  in intel_runtime_pm_put_raw()
327  * @rpm: the intel_runtime_pm structure
337 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)  in intel_runtime_pm_put_unchecked()  argument
339 	__intel_runtime_pm_put(rpm, INTEL_WAKEREF_DEF, true);  in intel_runtime_pm_put_unchecked()
345  * @rpm: the intel_runtime_pm structure
352 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put()  argument
354 	__intel_runtime_pm_put(rpm, wref, true);  in intel_runtime_pm_put()
360  * @rpm: the intel_runtime_pm structure
368 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_enable()  argument
370 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_enable()
371 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_enable()
387 	 * Take a permanent reference to disable the RPM functionality and drop  in intel_runtime_pm_enable()
389 	 * so the driver's own RPM reference tracking asserts also work on  in intel_runtime_pm_enable()
390 	 * platforms without RPM support.  in intel_runtime_pm_enable()
392 	if (!rpm->available) {  in intel_runtime_pm_enable()
414 	 * The core calls the driver load handler with an RPM reference held.  in intel_runtime_pm_enable()
421 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_disable()  argument
423 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_disable()
424 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_disable()
426 	/* Transfer rpm ownership back to core */  in intel_runtime_pm_disable()
428 		 "Failed to pass rpm ownership back to core\n");  in intel_runtime_pm_disable()
432 	if (!rpm->available)  in intel_runtime_pm_disable()
436 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_release()  argument
438 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_driver_release()
439 	int count = atomic_read(&rpm->wakeref_count);  in intel_runtime_pm_driver_release()
441 	intel_wakeref_auto_fini(&rpm->userfault_wakeref);  in intel_runtime_pm_driver_release()
449 void intel_runtime_pm_driver_last_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_last_release()  argument
451 	intel_runtime_pm_driver_release(rpm);  in intel_runtime_pm_driver_last_release()
452 	untrack_all_intel_runtime_pm_wakerefs(rpm);  in intel_runtime_pm_driver_last_release()
455 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)  in intel_runtime_pm_init_early()  argument
457 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_init_early()
461 	rpm->kdev = kdev;  in intel_runtime_pm_init_early()
462 	rpm->available = HAS_RUNTIME_PM(i915);  in intel_runtime_pm_init_early()
463 	atomic_set(&rpm->wakeref_count, 0);  in intel_runtime_pm_init_early()
465 	init_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_init_early()
466 	INIT_LIST_HEAD(&rpm->lmem_userfault_list);  in intel_runtime_pm_init_early()
467 	spin_lock_init(&rpm->lmem_userfault_lock);  in intel_runtime_pm_init_early()
468 	intel_wakeref_auto_init(&rpm->userfault_wakeref, i915);  in intel_runtime_pm_init_early()