1/*
2 * Copyright © 2012-2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 * Daniel Vetter <daniel.vetter@ffwll.ch>
26 *
27 */
28
29#include <linux/pm_runtime.h>
30
31#include <drm/drm_print.h>
32
33#include "i915_drv.h"
34#include "i915_trace.h"
35
36/**
37 * DOC: runtime pm
38 *
39 * The i915 driver supports dynamic enabling and disabling of entire hardware
40 * blocks at runtime. This is especially important on the display side where
41 * software is supposed to control many power gates manually on recent hardware,
42 * since on the GT side a lot of the power management is done by the hardware.
43 * But even there some manual control at the device level is required.
44 *
45 * Since i915 supports a diverse set of platforms with a unified codebase and
46 * hardware engineers just love to shuffle functionality around between power
47 * domains there's a sizeable amount of indirection required. This file provides
48 * generic functions to the driver for grabbing and releasing references for
49 * abstract power domains. It then maps those to the actual power wells
50 * present for a given platform.
51 */
52
53static struct drm_i915_private *rpm_to_i915(struct intel_runtime_pm *rpm)
54{
55 return container_of(rpm, struct drm_i915_private, runtime_pm);
56}
57
58#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
59
60static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)
61{
62 if (!rpm->debug.class)
63 ref_tracker_dir_init(&rpm->debug, INTEL_REFTRACK_DEAD_COUNT,
64 "intel_runtime_pm");
65}
66
67static intel_wakeref_t
68track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)
69{
70 if (!rpm->available || rpm->no_wakeref_tracking)
71 return INTEL_WAKEREF_DEF;
72
73 return intel_ref_tracker_alloc(&rpm->debug);
74}
75
76static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,
77 intel_wakeref_t wakeref)
78{
79 if (!rpm->available || rpm->no_wakeref_tracking)
80 return;
81
82 intel_ref_tracker_free(&rpm->debug, wakeref);
83}
84
85static void untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)
86{
87 ref_tracker_dir_exit(&rpm->debug);
88}
89
90static noinline void
91__intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)
92{
93 unsigned long flags;
94
95 if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,
96 &rpm->debug.lock,
97 flags))
98 return;
99
100 ref_tracker_dir_print_locked(&rpm->debug, INTEL_REFTRACK_PRINT_LIMIT);
101 spin_unlock_irqrestore(&rpm->debug.lock, flags);
102}
103
104void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,
105 struct drm_printer *p)
106{
107 intel_ref_tracker_show(&rpm->debug, p);
108}
109
110#else
111
112static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)
113{
114}
115
116static intel_wakeref_t
117track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)
118{
119 return INTEL_WAKEREF_DEF;
120}
121
122static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,
123 intel_wakeref_t wakeref)
124{
125}
126
127static void
128__intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)
129{
130 atomic_dec(v: &rpm->wakeref_count);
131}
132
133static void
134untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)
135{
136}
137
138#endif
139
140static void
141intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)
142{
143 if (wakelock) {
144 atomic_add(i: 1 + INTEL_RPM_WAKELOCK_BIAS, v: &rpm->wakeref_count);
145 assert_rpm_wakelock_held(rpm);
146 } else {
147 atomic_inc(v: &rpm->wakeref_count);
148 assert_rpm_raw_wakeref_held(rpm);
149 }
150}
151
152static void
153intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)
154{
155 if (wakelock) {
156 assert_rpm_wakelock_held(rpm);
157 atomic_sub(INTEL_RPM_WAKELOCK_BIAS, v: &rpm->wakeref_count);
158 } else {
159 assert_rpm_raw_wakeref_held(rpm);
160 }
161
162 __intel_wakeref_dec_and_check_tracking(rpm);
163}
164
165static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,
166 bool wakelock)
167{
168 struct drm_i915_private *i915 = rpm_to_i915(rpm);
169 int ret;
170
171 ret = pm_runtime_get_sync(dev: rpm->kdev);
172 drm_WARN_ONCE(&i915->drm, ret < 0,
173 "pm_runtime_get_sync() failed: %d\n", ret);
174
175 intel_runtime_pm_acquire(rpm, wakelock);
176
177 return track_intel_runtime_pm_wakeref(rpm);
178}
179
180/**
181 * intel_runtime_pm_get_raw - grab a raw runtime pm reference
182 * @rpm: the intel_runtime_pm structure
183 *
184 * This is the unlocked version of intel_display_power_is_enabled() and should
185 * only be used from error capture and recovery code where deadlocks are
186 * possible.
187 * This function grabs a device-level runtime pm reference (mostly used for
188 * asynchronous PM management from display code) and ensures that it is powered
189 * up. Raw references are not considered during wakelock assert checks.
190 *
191 * Any runtime pm reference obtained by this function must have a symmetric
192 * call to intel_runtime_pm_put_raw() to release the reference again.
193 *
194 * Returns: the wakeref cookie to pass to intel_runtime_pm_put_raw(), evaluates
195 * as True if the wakeref was acquired, or False otherwise.
196 */
197intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)
198{
199 return __intel_runtime_pm_get(rpm, wakelock: false);
200}
201
202/**
203 * intel_runtime_pm_get - grab a runtime pm reference
204 * @rpm: the intel_runtime_pm structure
205 *
206 * This function grabs a device-level runtime pm reference (mostly used for GEM
207 * code to ensure the GTT or GT is on) and ensures that it is powered up.
208 *
209 * Any runtime pm reference obtained by this function must have a symmetric
210 * call to intel_runtime_pm_put() to release the reference again.
211 *
212 * Returns: the wakeref cookie to pass to intel_runtime_pm_put()
213 */
214intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)
215{
216 return __intel_runtime_pm_get(rpm, wakelock: true);
217}
218
219/**
220 * __intel_runtime_pm_get_if_active - grab a runtime pm reference if device is active
221 * @rpm: the intel_runtime_pm structure
222 * @ignore_usecount: get a ref even if dev->power.usage_count is 0
223 *
224 * This function grabs a device-level runtime pm reference if the device is
225 * already active and ensures that it is powered up. It is illegal to try
226 * and access the HW should intel_runtime_pm_get_if_active() report failure.
227 *
228 * If @ignore_usecount is true, a reference will be acquired even if there is no
229 * user requiring the device to be powered up (dev->power.usage_count == 0).
230 * If the function returns false in this case then it's guaranteed that the
231 * device's runtime suspend hook has been called already or that it will be
232 * called (and hence it's also guaranteed that the device's runtime resume
233 * hook will be called eventually).
234 *
235 * Any runtime pm reference obtained by this function must have a symmetric
236 * call to intel_runtime_pm_put() to release the reference again.
237 *
238 * Returns: the wakeref cookie to pass to intel_runtime_pm_put(), evaluates
239 * as True if the wakeref was acquired, or False otherwise.
240 */
241static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm,
242 bool ignore_usecount)
243{
244 if (IS_ENABLED(CONFIG_PM)) {
245 /*
246 * In cases runtime PM is disabled by the RPM core and we get
247 * an -EINVAL return value we are not supposed to call this
248 * function, since the power state is undefined. This applies
249 * atm to the late/early system suspend/resume handlers.
250 */
251 if ((ignore_usecount &&
252 pm_runtime_get_if_active(dev: rpm->kdev) <= 0) ||
253 (!ignore_usecount &&
254 pm_runtime_get_if_in_use(dev: rpm->kdev) <= 0))
255 return NULL;
256 }
257
258 intel_runtime_pm_acquire(rpm, wakelock: true);
259
260 return track_intel_runtime_pm_wakeref(rpm);
261}
262
263intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)
264{
265 return __intel_runtime_pm_get_if_active(rpm, ignore_usecount: false);
266}
267
268intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)
269{
270 return __intel_runtime_pm_get_if_active(rpm, ignore_usecount: true);
271}
272
273/**
274 * intel_runtime_pm_get_noresume - grab a runtime pm reference
275 * @rpm: the intel_runtime_pm structure
276 *
277 * This function grabs a device-level runtime pm reference.
278 *
279 * It will _not_ resume the device but instead only get an extra wakeref.
280 * Therefore it is only valid to call this functions from contexts where
281 * the device is known to be active and with another wakeref previously hold.
282 *
283 * Any runtime pm reference obtained by this function must have a symmetric
284 * call to intel_runtime_pm_put() to release the reference again.
285 *
286 * Returns: the wakeref cookie to pass to intel_runtime_pm_put()
287 */
288intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)
289{
290 assert_rpm_raw_wakeref_held(rpm);
291 pm_runtime_get_noresume(dev: rpm->kdev);
292
293 intel_runtime_pm_acquire(rpm, wakelock: true);
294
295 return track_intel_runtime_pm_wakeref(rpm);
296}
297
298static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,
299 intel_wakeref_t wref,
300 bool wakelock)
301{
302 struct device *kdev = rpm->kdev;
303
304 untrack_intel_runtime_pm_wakeref(rpm, wakeref: wref);
305
306 intel_runtime_pm_release(rpm, wakelock);
307
308 pm_runtime_mark_last_busy(dev: kdev);
309 pm_runtime_put_autosuspend(dev: kdev);
310}
311
312/**
313 * intel_runtime_pm_put_raw - release a raw runtime pm reference
314 * @rpm: the intel_runtime_pm structure
315 * @wref: wakeref acquired for the reference that is being released
316 *
317 * This function drops the device-level runtime pm reference obtained by
318 * intel_runtime_pm_get_raw() and might power down the corresponding
319 * hardware block right away if this is the last reference.
320 */
321void
322intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)
323{
324 __intel_runtime_pm_put(rpm, wref, wakelock: false);
325}
326
327/**
328 * intel_runtime_pm_put_unchecked - release an unchecked runtime pm reference
329 * @rpm: the intel_runtime_pm structure
330 *
331 * This function drops the device-level runtime pm reference obtained by
332 * intel_runtime_pm_get() and might power down the corresponding
333 * hardware block right away if this is the last reference.
334 *
335 * This function exists only for historical reasons and should be avoided in
336 * new code, as the correctness of its use cannot be checked. Always use
337 * intel_runtime_pm_put() instead.
338 */
339void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)
340{
341 __intel_runtime_pm_put(rpm, INTEL_WAKEREF_DEF, wakelock: true);
342}
343
344#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
345/**
346 * intel_runtime_pm_put - release a runtime pm reference
347 * @rpm: the intel_runtime_pm structure
348 * @wref: wakeref acquired for the reference that is being released
349 *
350 * This function drops the device-level runtime pm reference obtained by
351 * intel_runtime_pm_get() and might power down the corresponding
352 * hardware block right away if this is the last reference.
353 */
354void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)
355{
356 __intel_runtime_pm_put(rpm, wref, true);
357}
358#endif
359
360/**
361 * intel_runtime_pm_enable - enable runtime pm
362 * @rpm: the intel_runtime_pm structure
363 *
364 * This function enables runtime pm at the end of the driver load sequence.
365 *
366 * Note that this function does currently not enable runtime pm for the
367 * subordinate display power domains. That is done by
368 * intel_power_domains_enable().
369 */
370void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)
371{
372 struct drm_i915_private *i915 = rpm_to_i915(rpm);
373 struct device *kdev = rpm->kdev;
374
375 /*
376 * Disable the system suspend direct complete optimization, which can
377 * leave the device suspended skipping the driver's suspend handlers
378 * if the device was already runtime suspended. This is needed due to
379 * the difference in our runtime and system suspend sequence and
380 * because the HDA driver may require us to enable the audio power
381 * domain during system suspend.
382 */
383 dev_pm_set_driver_flags(dev: kdev, DPM_FLAG_NO_DIRECT_COMPLETE);
384
385 pm_runtime_set_autosuspend_delay(dev: kdev, delay: 10000); /* 10s */
386 pm_runtime_mark_last_busy(dev: kdev);
387
388 /*
389 * Take a permanent reference to disable the RPM functionality and drop
390 * it only when unloading the driver. Use the low level get/put helpers,
391 * so the driver's own RPM reference tracking asserts also work on
392 * platforms without RPM support.
393 */
394 if (!rpm->available) {
395 int ret;
396
397 pm_runtime_dont_use_autosuspend(dev: kdev);
398 ret = pm_runtime_get_sync(dev: kdev);
399 drm_WARN(&i915->drm, ret < 0,
400 "pm_runtime_get_sync() failed: %d\n", ret);
401 } else {
402 pm_runtime_use_autosuspend(dev: kdev);
403 }
404
405 /*
406 * FIXME: Temp hammer to keep autosupend disable on lmem supported platforms.
407 * As per PCIe specs 5.3.1.4.1, all iomem read write request over a PCIe
408 * function will be unsupported in case PCIe endpoint function is in D3.
409 * Let's keep i915 autosuspend control 'on' till we fix all known issue
410 * with lmem access in D3.
411 */
412 if (!IS_DGFX(i915))
413 pm_runtime_allow(dev: kdev);
414
415 /*
416 * The core calls the driver load handler with an RPM reference held.
417 * We drop that here and will reacquire it during unloading in
418 * intel_power_domains_fini().
419 */
420 pm_runtime_put_autosuspend(dev: kdev);
421}
422
423void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)
424{
425 struct drm_i915_private *i915 = rpm_to_i915(rpm);
426 struct device *kdev = rpm->kdev;
427
428 /* Transfer rpm ownership back to core */
429 drm_WARN(&i915->drm, pm_runtime_get_sync(kdev) < 0,
430 "Failed to pass rpm ownership back to core\n");
431
432 pm_runtime_dont_use_autosuspend(dev: kdev);
433
434 if (!rpm->available)
435 pm_runtime_put(dev: kdev);
436}
437
438void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)
439{
440 struct drm_i915_private *i915 = rpm_to_i915(rpm);
441 int count = atomic_read(v: &rpm->wakeref_count);
442
443 intel_wakeref_auto_fini(wf: &rpm->userfault_wakeref);
444
445 drm_WARN(&i915->drm, count,
446 "i915 raw-wakerefs=%d wakelocks=%d on cleanup\n",
447 intel_rpm_raw_wakeref_count(count),
448 intel_rpm_wakelock_count(count));
449}
450
451void intel_runtime_pm_driver_last_release(struct intel_runtime_pm *rpm)
452{
453 intel_runtime_pm_driver_release(rpm);
454 untrack_all_intel_runtime_pm_wakerefs(rpm);
455}
456
457void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)
458{
459 struct drm_i915_private *i915 = rpm_to_i915(rpm);
460 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
461 struct device *kdev = &pdev->dev;
462
463 rpm->kdev = kdev;
464 rpm->available = HAS_RUNTIME_PM(i915);
465 atomic_set(v: &rpm->wakeref_count, i: 0);
466
467 init_intel_runtime_pm_wakeref(rpm);
468 INIT_LIST_HEAD(list: &rpm->lmem_userfault_list);
469 spin_lock_init(&rpm->lmem_userfault_lock);
470 intel_wakeref_auto_init(wf: &rpm->userfault_wakeref, i915);
471}
472