1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7#include <linux/wait_bit.h>
8
9#include "intel_runtime_pm.h"
10#include "intel_wakeref.h"
11#include "i915_drv.h"
12
13int __intel_wakeref_get_first(struct intel_wakeref *wf)
14{
15 intel_wakeref_t wakeref;
16 int ret = 0;
17
18 wakeref = intel_runtime_pm_get(rpm: &wf->i915->runtime_pm);
19 /*
20 * Treat get/put as different subclasses, as we may need to run
21 * the put callback from under the shrinker and do not want to
22 * cross-contanimate that callback with any extra work performed
23 * upon acquiring the wakeref.
24 */
25 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
26
27 if (!atomic_read(v: &wf->count)) {
28 INTEL_WAKEREF_BUG_ON(wf->wakeref);
29 wf->wakeref = wakeref;
30 wakeref = NULL;
31
32 ret = wf->ops->get(wf);
33 if (ret) {
34 wakeref = xchg(&wf->wakeref, NULL);
35 wake_up_var(var: &wf->wakeref);
36 goto unlock;
37 }
38
39 smp_mb__before_atomic(); /* release wf->count */
40 }
41
42 atomic_inc(v: &wf->count);
43 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
44
45unlock:
46 mutex_unlock(lock: &wf->mutex);
47 if (unlikely(wakeref))
48 intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref);
49
50 return ret;
51}
52
53static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
54{
55 intel_wakeref_t wakeref = NULL;
56
57 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
58 if (unlikely(!atomic_dec_and_test(&wf->count)))
59 goto unlock;
60
61 /* ops->put() must reschedule its own release on error/deferral */
62 if (likely(!wf->ops->put(wf))) {
63 INTEL_WAKEREF_BUG_ON(!wf->wakeref);
64 wakeref = xchg(&wf->wakeref, NULL);
65 wake_up_var(var: &wf->wakeref);
66 }
67
68unlock:
69 mutex_unlock(lock: &wf->mutex);
70 if (wakeref)
71 intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref);
72}
73
74void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
75{
76 INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work));
77
78 /* Assume we are not in process context and so cannot sleep. */
79 if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(lock: &wf->mutex)) {
80 mod_delayed_work(wq: wf->i915->unordered_wq, dwork: &wf->work,
81 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));
82 return;
83 }
84
85 ____intel_wakeref_put_last(wf);
86}
87
88static void __intel_wakeref_put_work(struct work_struct *wrk)
89{
90 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work);
91
92 if (atomic_add_unless(v: &wf->count, a: -1, u: 1))
93 return;
94
95 mutex_lock(lock: &wf->mutex);
96 ____intel_wakeref_put_last(wf);
97}
98
99void __intel_wakeref_init(struct intel_wakeref *wf,
100 struct drm_i915_private *i915,
101 const struct intel_wakeref_ops *ops,
102 struct intel_wakeref_lockclass *key,
103 const char *name)
104{
105 wf->i915 = i915;
106 wf->ops = ops;
107
108 __mutex_init(lock: &wf->mutex, name: "wakeref.mutex", key: &key->mutex);
109 atomic_set(v: &wf->count, i: 0);
110 wf->wakeref = NULL;
111
112 INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work);
113 lockdep_init_map(&wf->work.work.lockdep_map,
114 "wakeref.work", &key->work, 0);
115
116#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_WAKEREF)
117 if (!wf->debug.class)
118 ref_tracker_dir_init(&wf->debug, INTEL_REFTRACK_DEAD_COUNT, "intel_wakeref");
119#endif
120}
121
122int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
123{
124 int err;
125
126 might_sleep();
127
128 err = wait_var_event_killable(&wf->wakeref,
129 !intel_wakeref_is_active(wf));
130 if (err)
131 return err;
132
133 intel_wakeref_unlock_wait(wf);
134 return 0;
135}
136
137static void wakeref_auto_timeout(struct timer_list *t)
138{
139 struct intel_wakeref_auto *wf = timer_container_of(wf, t, timer);
140 intel_wakeref_t wakeref;
141 unsigned long flags;
142
143 if (!refcount_dec_and_lock_irqsave(r: &wf->count, lock: &wf->lock, flags: &flags))
144 return;
145
146 wakeref = xchg(&wf->wakeref, NULL);
147 spin_unlock_irqrestore(lock: &wf->lock, flags);
148
149 intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref);
150}
151
152void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
153 struct drm_i915_private *i915)
154{
155 spin_lock_init(&wf->lock);
156 timer_setup(&wf->timer, wakeref_auto_timeout, 0);
157 refcount_set(r: &wf->count, n: 0);
158 wf->wakeref = NULL;
159 wf->i915 = i915;
160}
161
162void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
163{
164 unsigned long flags;
165
166 if (!timeout) {
167 if (timer_delete_sync(timer: &wf->timer))
168 wakeref_auto_timeout(t: &wf->timer);
169 return;
170 }
171
172 /* Our mission is that we only extend an already active wakeref */
173 assert_rpm_wakelock_held(rpm: &wf->i915->runtime_pm);
174
175 if (!refcount_inc_not_zero(r: &wf->count)) {
176 spin_lock_irqsave(&wf->lock, flags);
177 if (!refcount_inc_not_zero(r: &wf->count)) {
178 INTEL_WAKEREF_BUG_ON(wf->wakeref);
179 wf->wakeref =
180 intel_runtime_pm_get_if_in_use(rpm: &wf->i915->runtime_pm);
181 refcount_set(r: &wf->count, n: 1);
182 }
183 spin_unlock_irqrestore(lock: &wf->lock, flags);
184 }
185
186 /*
187 * If we extend a pending timer, we will only get a single timer
188 * callback and so need to cancel the local inc by running the
189 * elided callback to keep the wf->count balanced.
190 */
191 if (mod_timer(timer: &wf->timer, expires: jiffies + timeout))
192 wakeref_auto_timeout(t: &wf->timer);
193}
194
195void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
196{
197 intel_wakeref_auto(wf, timeout: 0);
198 INTEL_WAKEREF_BUG_ON(wf->wakeref);
199}
200
201void intel_ref_tracker_show(struct ref_tracker_dir *dir,
202 struct drm_printer *p)
203{
204 const size_t buf_size = PAGE_SIZE;
205 char *buf, *sb, *se;
206 size_t count;
207
208 buf = kmalloc(buf_size, GFP_NOWAIT);
209 if (!buf)
210 return;
211
212 count = ref_tracker_dir_snprint(dir, buf, size: buf_size);
213 if (!count)
214 goto free;
215 /* printk does not like big buffers, so we split it */
216 for (sb = buf; *sb; sb = se + 1) {
217 se = strchrnul(sb, '\n');
218 drm_printf(p, f: "%.*s", (int)(se - sb + 1), sb);
219 if (!*se)
220 break;
221 }
222 if (count >= buf_size)
223 drm_printf(p, f: "\n...dropped %zd extra bytes of leak report.\n",
224 count + 1 - buf_size);
225free:
226 kfree(objp: buf);
227}
228