1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <linux/string_helpers.h>
7
8#include <drm/intel/i915_drm.h>
9
10#include "display/intel_display.h"
11#include "display/intel_display_rps.h"
12#include "soc/intel_dram.h"
13
14#include "i915_drv.h"
15#include "i915_irq.h"
16#include "i915_reg.h"
17#include "i915_wait_util.h"
18#include "intel_breadcrumbs.h"
19#include "intel_gt.h"
20#include "intel_gt_clock_utils.h"
21#include "intel_gt_irq.h"
22#include "intel_gt_pm.h"
23#include "intel_gt_pm_irq.h"
24#include "intel_gt_print.h"
25#include "intel_gt_regs.h"
26#include "intel_mchbar_regs.h"
27#include "intel_pcode.h"
28#include "intel_rps.h"
29#include "vlv_iosf_sb.h"
30#include "../../../platform/x86/intel_ips.h"
31
32#define BUSY_MAX_EI 20u /* ms */
33
34/*
35 * Lock protecting IPS related data structures
36 */
37static DEFINE_SPINLOCK(mchdev_lock);
38
39static struct intel_gt *rps_to_gt(struct intel_rps *rps)
40{
41 return container_of(rps, struct intel_gt, rps);
42}
43
44static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
45{
46 return rps_to_gt(rps)->i915;
47}
48
49static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
50{
51 return rps_to_gt(rps)->uncore;
52}
53
54static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
55{
56 struct intel_gt *gt = rps_to_gt(rps);
57
58 return &gt_to_guc(gt)->slpc;
59}
60
61static bool rps_uses_slpc(struct intel_rps *rps)
62{
63 struct intel_gt *gt = rps_to_gt(rps);
64
65 return intel_uc_uses_guc_slpc(uc: &gt->uc);
66}
67
68static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
69{
70 return mask & ~rps->pm_intrmsk_mbz;
71}
72
73static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
74{
75 intel_uncore_write_fw(uncore, reg, val);
76}
77
78static void rps_timer(struct timer_list *t)
79{
80 struct intel_rps *rps = timer_container_of(rps, t, timer);
81 struct intel_gt *gt = rps_to_gt(rps);
82 struct intel_engine_cs *engine;
83 ktime_t dt, last, timestamp;
84 enum intel_engine_id id;
85 s64 max_busy[3] = {};
86
87 timestamp = 0;
88 for_each_engine(engine, gt, id) {
89 s64 busy;
90 int i;
91
92 dt = intel_engine_get_busy_time(engine, now: &timestamp);
93 last = engine->stats.rps;
94 engine->stats.rps = dt;
95
96 busy = ktime_to_ns(ktime_sub(dt, last));
97 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
98 if (busy > max_busy[i])
99 swap(busy, max_busy[i]);
100 }
101 }
102 last = rps->pm_timestamp;
103 rps->pm_timestamp = timestamp;
104
105 if (intel_rps_is_active(rps)) {
106 s64 busy;
107 int i;
108
109 dt = ktime_sub(timestamp, last);
110
111 /*
112 * Our goal is to evaluate each engine independently, so we run
113 * at the lowest clocks required to sustain the heaviest
114 * workload. However, a task may be split into sequential
115 * dependent operations across a set of engines, such that
116 * the independent contributions do not account for high load,
117 * but overall the task is GPU bound. For example, consider
118 * video decode on vcs followed by colour post-processing
119 * on vecs, followed by general post-processing on rcs.
120 * Since multi-engines being active does imply a single
121 * continuous workload across all engines, we hedge our
122 * bets by only contributing a factor of the distributed
123 * load into our busyness calculation.
124 */
125 busy = max_busy[0];
126 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
127 if (!max_busy[i])
128 break;
129
130 busy += div_u64(dividend: max_busy[i], divisor: 1 << i);
131 }
132 GT_TRACE(gt,
133 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
134 busy, (int)div64_u64(100 * busy, dt),
135 max_busy[0], max_busy[1], max_busy[2],
136 rps->pm_interval);
137
138 if (100 * busy > rps->power.up_threshold * dt &&
139 rps->cur_freq < rps->max_freq_softlimit) {
140 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
141 rps->pm_interval = 1;
142 queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
143 } else if (100 * busy < rps->power.down_threshold * dt &&
144 rps->cur_freq > rps->min_freq_softlimit) {
145 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
146 rps->pm_interval = 1;
147 queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
148 } else {
149 rps->last_adj = 0;
150 }
151
152 mod_timer(timer: &rps->timer,
153 expires: jiffies + msecs_to_jiffies(m: rps->pm_interval));
154 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
155 }
156}
157
158static void rps_start_timer(struct intel_rps *rps)
159{
160 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
161 rps->pm_interval = 1;
162 mod_timer(timer: &rps->timer, expires: jiffies + 1);
163}
164
165static void rps_stop_timer(struct intel_rps *rps)
166{
167 timer_delete_sync(timer: &rps->timer);
168 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
169 cancel_work_sync(work: &rps->work);
170}
171
172static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
173{
174 u32 mask = 0;
175
176 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
177 if (val > rps->min_freq_softlimit)
178 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
179 GEN6_PM_RP_DOWN_THRESHOLD |
180 GEN6_PM_RP_DOWN_TIMEOUT);
181
182 if (val < rps->max_freq_softlimit)
183 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
184
185 mask &= rps->pm_events;
186
187 return rps_pm_sanitize_mask(rps, mask: ~mask);
188}
189
190static void rps_reset_ei(struct intel_rps *rps)
191{
192 memset(s: &rps->ei, c: 0, n: sizeof(rps->ei));
193}
194
195static void rps_enable_interrupts(struct intel_rps *rps)
196{
197 struct intel_gt *gt = rps_to_gt(rps);
198
199 GEM_BUG_ON(rps_uses_slpc(rps));
200
201 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
202 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
203
204 rps_reset_ei(rps);
205
206 spin_lock_irq(lock: gt->irq_lock);
207 gen6_gt_pm_enable_irq(gt, enable_mask: rps->pm_events);
208 spin_unlock_irq(lock: gt->irq_lock);
209
210 intel_uncore_write(uncore: gt->uncore,
211 GEN6_PMINTRMSK, val: rps_pm_mask(rps, val: rps->last_freq));
212}
213
214static void gen6_rps_reset_interrupts(struct intel_rps *rps)
215{
216 gen6_gt_pm_reset_iir(gt: rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
217}
218
219static void gen11_rps_reset_interrupts(struct intel_rps *rps)
220{
221 while (gen11_gt_reset_one_iir(gt: rps_to_gt(rps), bank: 0, GEN11_GTPM))
222 ;
223}
224
225static void rps_reset_interrupts(struct intel_rps *rps)
226{
227 struct intel_gt *gt = rps_to_gt(rps);
228
229 spin_lock_irq(lock: gt->irq_lock);
230 if (GRAPHICS_VER(gt->i915) >= 11)
231 gen11_rps_reset_interrupts(rps);
232 else
233 gen6_rps_reset_interrupts(rps);
234
235 rps->pm_iir = 0;
236 spin_unlock_irq(lock: gt->irq_lock);
237}
238
239static void rps_disable_interrupts(struct intel_rps *rps)
240{
241 struct intel_gt *gt = rps_to_gt(rps);
242
243 intel_uncore_write(uncore: gt->uncore,
244 GEN6_PMINTRMSK, val: rps_pm_sanitize_mask(rps, mask: ~0u));
245
246 spin_lock_irq(lock: gt->irq_lock);
247 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
248 spin_unlock_irq(lock: gt->irq_lock);
249
250 intel_synchronize_irq(i915: gt->i915);
251
252 /*
253 * Now that we will not be generating any more work, flush any
254 * outstanding tasks. As we are called on the RPS idle path,
255 * we will reset the GPU to minimum frequencies, so the current
256 * state of the worker can be discarded.
257 */
258 cancel_work_sync(work: &rps->work);
259
260 rps_reset_interrupts(rps);
261 GT_TRACE(gt, "interrupts:off\n");
262}
263
264static const struct cparams {
265 u16 i;
266 u16 t;
267 u16 m;
268 u16 c;
269} cparams[] = {
270 { 1, 1333, 301, 28664 },
271 { 1, 1067, 294, 24460 },
272 { 1, 800, 294, 25192 },
273 { 0, 1333, 276, 27605 },
274 { 0, 1067, 276, 27605 },
275 { 0, 800, 231, 23784 },
276};
277
278static void gen5_rps_init(struct intel_rps *rps)
279{
280 struct drm_i915_private *i915 = rps_to_i915(rps);
281 struct intel_uncore *uncore = rps_to_uncore(rps);
282 unsigned int fsb_freq, mem_freq;
283 u8 fmax, fmin, fstart;
284 u32 rgvmodectl;
285 int c_m, i;
286
287 fsb_freq = intel_fsb_freq(i915);
288 mem_freq = intel_mem_freq(i915);
289
290 if (fsb_freq <= 3200000)
291 c_m = 0;
292 else if (fsb_freq <= 4800000)
293 c_m = 1;
294 else
295 c_m = 2;
296
297 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
298 if (cparams[i].i == c_m &&
299 cparams[i].t == DIV_ROUND_CLOSEST(mem_freq, 1000)) {
300 rps->ips.m = cparams[i].m;
301 rps->ips.c = cparams[i].c;
302 break;
303 }
304 }
305
306 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
307
308 /* Set up min, max, and cur for interrupt handling */
309 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
310 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
311 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
312 MEMMODE_FSTART_SHIFT;
313 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
314 fmax, fmin, fstart);
315
316 rps->min_freq = fmax;
317 rps->efficient_freq = fstart;
318 rps->max_freq = fmin;
319}
320
321static unsigned long
322__ips_chipset_val(struct intel_ips *ips)
323{
324 struct intel_uncore *uncore =
325 rps_to_uncore(container_of(ips, struct intel_rps, ips));
326 unsigned long now = jiffies_to_msecs(j: jiffies), dt;
327 unsigned long result;
328 u64 total, delta;
329
330 lockdep_assert_held(&mchdev_lock);
331
332 /*
333 * Prevent division-by-zero if we are asking too fast.
334 * Also, we don't get interesting results if we are polling
335 * faster than once in 10ms, so just return the saved value
336 * in such cases.
337 */
338 dt = now - ips->last_time1;
339 if (dt <= 10)
340 return ips->chipset_power;
341
342 /* FIXME: handle per-counter overflow */
343 total = intel_uncore_read(uncore, DMIEC);
344 total += intel_uncore_read(uncore, DDREC);
345 total += intel_uncore_read(uncore, CSIEC);
346
347 delta = total - ips->last_count1;
348
349 result = div_u64(dividend: div_u64(dividend: ips->m * delta, divisor: dt) + ips->c, divisor: 10);
350
351 ips->last_count1 = total;
352 ips->last_time1 = now;
353
354 ips->chipset_power = result;
355
356 return result;
357}
358
359static unsigned long ips_mch_val(struct intel_uncore *uncore)
360{
361 unsigned int m, x, b;
362 u32 tsfs;
363
364 tsfs = intel_uncore_read(uncore, TSFS);
365 x = intel_uncore_read8(uncore, TR1);
366
367 b = tsfs & TSFS_INTR_MASK;
368 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
369
370 return m * x / 127 - b;
371}
372
373static int _pxvid_to_vd(u8 pxvid)
374{
375 if (pxvid == 0)
376 return 0;
377
378 if (pxvid >= 8 && pxvid < 31)
379 pxvid = 31;
380
381 return (pxvid + 2) * 125;
382}
383
384static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
385{
386 const int vd = _pxvid_to_vd(pxvid);
387
388 if (INTEL_INFO(i915)->is_mobile)
389 return max(vd - 1125, 0);
390
391 return vd;
392}
393
394static void __gen5_ips_update(struct intel_ips *ips)
395{
396 struct intel_uncore *uncore =
397 rps_to_uncore(container_of(ips, struct intel_rps, ips));
398 u64 now, delta, dt;
399 u32 count;
400
401 lockdep_assert_held(&mchdev_lock);
402
403 now = ktime_get_raw_ns();
404 dt = now - ips->last_time2;
405 do_div(dt, NSEC_PER_MSEC);
406
407 /* Don't divide by 0 */
408 if (dt <= 10)
409 return;
410
411 count = intel_uncore_read(uncore, GFXEC);
412 delta = count - ips->last_count2;
413
414 ips->last_count2 = count;
415 ips->last_time2 = now;
416
417 /* More magic constants... */
418 ips->gfx_power = div_u64(dividend: delta * 1181, divisor: dt * 10);
419}
420
421static void gen5_rps_update(struct intel_rps *rps)
422{
423 spin_lock_irq(lock: &mchdev_lock);
424 __gen5_ips_update(ips: &rps->ips);
425 spin_unlock_irq(lock: &mchdev_lock);
426}
427
428static unsigned int gen5_invert_freq(struct intel_rps *rps,
429 unsigned int val)
430{
431 /* Invert the frequency bin into an ips delay */
432 val = rps->max_freq - val;
433 val = rps->min_freq + val;
434
435 return val;
436}
437
438static int __gen5_rps_set(struct intel_rps *rps, u8 val)
439{
440 struct intel_uncore *uncore = rps_to_uncore(rps);
441 u16 rgvswctl;
442
443 lockdep_assert_held(&mchdev_lock);
444
445 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
446 if (rgvswctl & MEMCTL_CMD_STS) {
447 drm_dbg(&rps_to_i915(rps)->drm,
448 "gpu busy, RCS change rejected\n");
449 return -EBUSY; /* still busy with another command */
450 }
451
452 /* Invert the frequency bin into an ips delay */
453 val = gen5_invert_freq(rps, val);
454
455 rgvswctl =
456 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
457 (val << MEMCTL_FREQ_SHIFT) |
458 MEMCTL_SFCAVM;
459 intel_uncore_write16(uncore, MEMSWCTL, val: rgvswctl);
460 intel_uncore_posting_read16(uncore, MEMSWCTL);
461
462 rgvswctl |= MEMCTL_CMD_STS;
463 intel_uncore_write16(uncore, MEMSWCTL, val: rgvswctl);
464
465 return 0;
466}
467
468static int gen5_rps_set(struct intel_rps *rps, u8 val)
469{
470 int err;
471
472 spin_lock_irq(lock: &mchdev_lock);
473 err = __gen5_rps_set(rps, val);
474 spin_unlock_irq(lock: &mchdev_lock);
475
476 return err;
477}
478
479static unsigned long intel_pxfreq(u32 vidfreq)
480{
481 int div = (vidfreq & 0x3f0000) >> 16;
482 int post = (vidfreq & 0x3000) >> 12;
483 int pre = (vidfreq & 0x7);
484
485 if (!pre)
486 return 0;
487
488 return div * 133333 / (pre << post);
489}
490
491static unsigned int init_emon(struct intel_uncore *uncore)
492{
493 u8 pxw[16];
494 int i;
495
496 /* Disable to program */
497 intel_uncore_write(uncore, ECR, val: 0);
498 intel_uncore_posting_read(uncore, ECR);
499
500 /* Program energy weights for various events */
501 intel_uncore_write(uncore, SDEW, val: 0x15040d00);
502 intel_uncore_write(uncore, CSIEW0, val: 0x007f0000);
503 intel_uncore_write(uncore, CSIEW1, val: 0x1e220004);
504 intel_uncore_write(uncore, CSIEW2, val: 0x04000004);
505
506 for (i = 0; i < 5; i++)
507 intel_uncore_write(uncore, PEW(i), val: 0);
508 for (i = 0; i < 3; i++)
509 intel_uncore_write(uncore, DEW(i), val: 0);
510
511 /* Program P-state weights to account for frequency power adjustment */
512 for (i = 0; i < 16; i++) {
513 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
514 unsigned int freq = intel_pxfreq(vidfreq: pxvidfreq);
515 unsigned int vid =
516 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
517 unsigned int val;
518
519 val = vid * vid * freq / 1000 * 255;
520 val /= 127 * 127 * 900;
521
522 pxw[i] = val;
523 }
524 /* Render standby states get 0 weight */
525 pxw[14] = 0;
526 pxw[15] = 0;
527
528 for (i = 0; i < 4; i++) {
529 intel_uncore_write(uncore, PXW(i),
530 val: pxw[i * 4 + 0] << 24 |
531 pxw[i * 4 + 1] << 16 |
532 pxw[i * 4 + 2] << 8 |
533 pxw[i * 4 + 3] << 0);
534 }
535
536 /* Adjust magic regs to magic values (more experimental results) */
537 intel_uncore_write(uncore, OGW0, val: 0);
538 intel_uncore_write(uncore, OGW1, val: 0);
539 intel_uncore_write(uncore, EG0, val: 0x00007f00);
540 intel_uncore_write(uncore, EG1, val: 0x0000000e);
541 intel_uncore_write(uncore, EG2, val: 0x000e0000);
542 intel_uncore_write(uncore, EG3, val: 0x68000300);
543 intel_uncore_write(uncore, EG4, val: 0x42000000);
544 intel_uncore_write(uncore, EG5, val: 0x00140031);
545 intel_uncore_write(uncore, EG6, val: 0);
546 intel_uncore_write(uncore, EG7, val: 0);
547
548 for (i = 0; i < 8; i++)
549 intel_uncore_write(uncore, PXWL(i), val: 0);
550
551 /* Enable PMON + select events */
552 intel_uncore_write(uncore, ECR, val: 0x80000019);
553
554 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
555}
556
557static bool gen5_rps_enable(struct intel_rps *rps)
558{
559 struct drm_i915_private *i915 = rps_to_i915(rps);
560 struct intel_display *display = i915->display;
561 struct intel_uncore *uncore = rps_to_uncore(rps);
562 u8 fstart, vstart;
563 u32 rgvmodectl;
564
565 spin_lock_irq(lock: &mchdev_lock);
566
567 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
568
569 /* Enable temp reporting */
570 intel_uncore_write16(uncore, PMMISC,
571 val: intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
572 intel_uncore_write16(uncore, TSC1,
573 val: intel_uncore_read16(uncore, TSC1) | TSE);
574
575 /* 100ms RC evaluation intervals */
576 intel_uncore_write(uncore, RCUPEI, val: 100000);
577 intel_uncore_write(uncore, RCDNEI, val: 100000);
578
579 /* Set max/min thresholds to 90ms and 80ms respectively */
580 intel_uncore_write(uncore, RCBMAXAVG, val: 90000);
581 intel_uncore_write(uncore, RCBMINAVG, val: 80000);
582
583 intel_uncore_write(uncore, MEMIHYST, val: 1);
584
585 /* Set up min, max, and cur for interrupt handling */
586 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
587 MEMMODE_FSTART_SHIFT;
588
589 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
590 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
591
592 intel_uncore_write(uncore,
593 MEMINTREN,
594 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
595
596 intel_uncore_write(uncore, VIDSTART, val: vstart);
597 intel_uncore_posting_read(uncore, VIDSTART);
598
599 rgvmodectl |= MEMMODE_SWMODE_EN;
600 intel_uncore_write(uncore, MEMMODECTL, val: rgvmodectl);
601
602 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
603 MEMCTL_CMD_STS) == 0, 10))
604 drm_err(&uncore->i915->drm,
605 "stuck trying to change perf mode\n");
606 mdelay(1);
607
608 __gen5_rps_set(rps, val: rps->cur_freq);
609
610 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
611 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
612 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
613 rps->ips.last_time1 = jiffies_to_msecs(j: jiffies);
614
615 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
616 rps->ips.last_time2 = ktime_get_raw_ns();
617
618 ilk_display_rps_enable(display);
619
620 spin_unlock_irq(lock: &mchdev_lock);
621
622 rps->ips.corr = init_emon(uncore);
623
624 return true;
625}
626
627static void gen5_rps_disable(struct intel_rps *rps)
628{
629 struct drm_i915_private *i915 = rps_to_i915(rps);
630 struct intel_display *display = i915->display;
631 struct intel_uncore *uncore = rps_to_uncore(rps);
632 u16 rgvswctl;
633
634 spin_lock_irq(lock: &mchdev_lock);
635
636 ilk_display_rps_disable(display);
637
638 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
639
640 /* Ack interrupts, disable EFC interrupt */
641 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, set: 0);
642 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
643
644 /* Go back to the starting frequency */
645 __gen5_rps_set(rps, val: rps->idle_freq);
646 mdelay(1);
647 rgvswctl |= MEMCTL_CMD_STS;
648 intel_uncore_write(uncore, MEMSWCTL, val: rgvswctl);
649 mdelay(1);
650
651 spin_unlock_irq(lock: &mchdev_lock);
652}
653
654static u32 rps_limits(struct intel_rps *rps, u8 val)
655{
656 u32 limits;
657
658 /*
659 * Only set the down limit when we've reached the lowest level to avoid
660 * getting more interrupts, otherwise leave this clear. This prevents a
661 * race in the hw when coming out of rc6: There's a tiny window where
662 * the hw runs at the minimal clock before selecting the desired
663 * frequency, if the down threshold expires in that window we will not
664 * receive a down interrupt.
665 */
666 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
667 limits = rps->max_freq_softlimit << 23;
668 if (val <= rps->min_freq_softlimit)
669 limits |= rps->min_freq_softlimit << 14;
670 } else {
671 limits = rps->max_freq_softlimit << 24;
672 if (val <= rps->min_freq_softlimit)
673 limits |= rps->min_freq_softlimit << 16;
674 }
675
676 return limits;
677}
678
679static void rps_set_power(struct intel_rps *rps, int new_power)
680{
681 struct intel_gt *gt = rps_to_gt(rps);
682 struct intel_uncore *uncore = gt->uncore;
683 u32 ei_up = 0, ei_down = 0;
684
685 lockdep_assert_held(&rps->power.mutex);
686
687 if (new_power == rps->power.mode)
688 return;
689
690 /* Note the units here are not exactly 1us, but 1280ns. */
691 switch (new_power) {
692 case LOW_POWER:
693 ei_up = 16000;
694 ei_down = 32000;
695 break;
696
697 case BETWEEN:
698 ei_up = 13000;
699 ei_down = 32000;
700 break;
701
702 case HIGH_POWER:
703 ei_up = 10000;
704 ei_down = 32000;
705 break;
706 }
707
708 /* When byt can survive without system hang with dynamic
709 * sw freq adjustments, this restriction can be lifted.
710 */
711 if (IS_VALLEYVIEW(gt->i915))
712 goto skip_hw_write;
713
714 GT_TRACE(gt,
715 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
716 new_power,
717 rps->power.up_threshold, ei_up,
718 rps->power.down_threshold, ei_down);
719
720 set(uncore, GEN6_RP_UP_EI,
721 val: intel_gt_ns_to_pm_interval(gt, ns: ei_up * 1000));
722 set(uncore, GEN6_RP_UP_THRESHOLD,
723 val: intel_gt_ns_to_pm_interval(gt,
724 ns: ei_up * rps->power.up_threshold * 10));
725
726 set(uncore, GEN6_RP_DOWN_EI,
727 val: intel_gt_ns_to_pm_interval(gt, ns: ei_down * 1000));
728 set(uncore, GEN6_RP_DOWN_THRESHOLD,
729 val: intel_gt_ns_to_pm_interval(gt,
730 ns: ei_down *
731 rps->power.down_threshold * 10));
732
733 set(uncore, GEN6_RP_CONTROL,
734 val: (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
735 GEN6_RP_MEDIA_HW_NORMAL_MODE |
736 GEN6_RP_MEDIA_IS_GFX |
737 GEN6_RP_ENABLE |
738 GEN6_RP_UP_BUSY_AVG |
739 GEN6_RP_DOWN_IDLE_AVG);
740
741skip_hw_write:
742 rps->power.mode = new_power;
743}
744
745static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
746{
747 int new_power;
748
749 new_power = rps->power.mode;
750 switch (rps->power.mode) {
751 case LOW_POWER:
752 if (val > rps->efficient_freq + 1 &&
753 val > rps->cur_freq)
754 new_power = BETWEEN;
755 break;
756
757 case BETWEEN:
758 if (val <= rps->efficient_freq &&
759 val < rps->cur_freq)
760 new_power = LOW_POWER;
761 else if (val >= rps->rp0_freq &&
762 val > rps->cur_freq)
763 new_power = HIGH_POWER;
764 break;
765
766 case HIGH_POWER:
767 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
768 val < rps->cur_freq)
769 new_power = BETWEEN;
770 break;
771 }
772 /* Max/min bins are special */
773 if (val <= rps->min_freq_softlimit)
774 new_power = LOW_POWER;
775 if (val >= rps->max_freq_softlimit)
776 new_power = HIGH_POWER;
777
778 mutex_lock(lock: &rps->power.mutex);
779 if (rps->power.interactive)
780 new_power = HIGH_POWER;
781 rps_set_power(rps, new_power);
782 mutex_unlock(lock: &rps->power.mutex);
783}
784
785void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
786{
787 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
788 str_yes_no(interactive));
789
790 mutex_lock(lock: &rps->power.mutex);
791 if (interactive) {
792 if (!rps->power.interactive++ && intel_rps_is_active(rps))
793 rps_set_power(rps, new_power: HIGH_POWER);
794 } else {
795 GEM_BUG_ON(!rps->power.interactive);
796 rps->power.interactive--;
797 }
798 mutex_unlock(lock: &rps->power.mutex);
799}
800
801static int gen6_rps_set(struct intel_rps *rps, u8 val)
802{
803 struct intel_uncore *uncore = rps_to_uncore(rps);
804 struct drm_i915_private *i915 = rps_to_i915(rps);
805 u32 swreq;
806
807 GEM_BUG_ON(rps_uses_slpc(rps));
808
809 if (GRAPHICS_VER(i915) >= 9)
810 swreq = GEN9_FREQUENCY(val);
811 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
812 swreq = HSW_FREQUENCY(val);
813 else
814 swreq = (GEN6_FREQUENCY(val) |
815 GEN6_OFFSET(0) |
816 GEN6_AGGRESSIVE_TURBO);
817 set(uncore, GEN6_RPNSWREQ, val: swreq);
818
819 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
820 val, intel_gpu_freq(rps, val), swreq);
821
822 return 0;
823}
824
825static int vlv_rps_set(struct intel_rps *rps, u8 val)
826{
827 struct drm_i915_private *i915 = rps_to_i915(rps);
828 int err;
829
830 vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
831 err = vlv_iosf_sb_write(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_REQ, val);
832 vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
833
834 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
835 val, intel_gpu_freq(rps, val));
836
837 return err;
838}
839
840static int rps_set(struct intel_rps *rps, u8 val, bool update)
841{
842 struct drm_i915_private *i915 = rps_to_i915(rps);
843 int err;
844
845 if (val == rps->last_freq)
846 return 0;
847
848 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
849 err = vlv_rps_set(rps, val);
850 else if (GRAPHICS_VER(i915) >= 6)
851 err = gen6_rps_set(rps, val);
852 else
853 err = gen5_rps_set(rps, val);
854 if (err)
855 return err;
856
857 if (update && GRAPHICS_VER(i915) >= 6)
858 gen6_rps_set_thresholds(rps, val);
859 rps->last_freq = val;
860
861 return 0;
862}
863
864void intel_rps_unpark(struct intel_rps *rps)
865{
866 if (!intel_rps_is_enabled(rps))
867 return;
868
869 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
870
871 /*
872 * Use the user's desired frequency as a guide, but for better
873 * performance, jump directly to RPe as our starting frequency.
874 */
875 mutex_lock(lock: &rps->lock);
876
877 intel_rps_set_active(rps);
878 intel_rps_set(rps,
879 clamp(rps->cur_freq,
880 rps->min_freq_softlimit,
881 rps->max_freq_softlimit));
882
883 mutex_unlock(lock: &rps->lock);
884
885 rps->pm_iir = 0;
886 if (intel_rps_has_interrupts(rps))
887 rps_enable_interrupts(rps);
888 if (intel_rps_uses_timer(rps))
889 rps_start_timer(rps);
890
891 if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
892 gen5_rps_update(rps);
893}
894
895void intel_rps_park(struct intel_rps *rps)
896{
897 int adj;
898
899 if (!intel_rps_is_enabled(rps))
900 return;
901
902 if (!intel_rps_clear_active(rps))
903 return;
904
905 if (intel_rps_uses_timer(rps))
906 rps_stop_timer(rps);
907 if (intel_rps_has_interrupts(rps))
908 rps_disable_interrupts(rps);
909
910 if (rps->last_freq <= rps->idle_freq)
911 return;
912
913 /*
914 * The punit delays the write of the frequency and voltage until it
915 * determines the GPU is awake. During normal usage we don't want to
916 * waste power changing the frequency if the GPU is sleeping (rc6).
917 * However, the GPU and driver is now idle and we do not want to delay
918 * switching to minimum voltage (reducing power whilst idle) as we do
919 * not expect to be woken in the near future and so must flush the
920 * change by waking the device.
921 *
922 * We choose to take the media powerwell (either would do to trick the
923 * punit into committing the voltage change) as that takes a lot less
924 * power than the render powerwell.
925 */
926 intel_uncore_forcewake_get(uncore: rps_to_uncore(rps), domains: FORCEWAKE_MEDIA);
927 rps_set(rps, val: rps->idle_freq, update: false);
928 intel_uncore_forcewake_put(uncore: rps_to_uncore(rps), domains: FORCEWAKE_MEDIA);
929
930 /*
931 * Since we will try and restart from the previously requested
932 * frequency on unparking, treat this idle point as a downclock
933 * interrupt and reduce the frequency for resume. If we park/unpark
934 * more frequently than the rps worker can run, we will not respond
935 * to any EI and never see a change in frequency.
936 *
937 * (Note we accommodate Cherryview's limitation of only using an
938 * even bin by applying it to all.)
939 */
940 adj = rps->last_adj;
941 if (adj < 0)
942 adj *= 2;
943 else /* CHV needs even encode values */
944 adj = -2;
945 rps->last_adj = adj;
946 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
947 if (rps->cur_freq < rps->efficient_freq) {
948 rps->cur_freq = rps->efficient_freq;
949 rps->last_adj = 0;
950 }
951
952 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
953}
954
955u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
956{
957 struct intel_guc_slpc *slpc;
958
959 if (rps_uses_slpc(rps)) {
960 slpc = rps_to_slpc(rps);
961
962 return slpc->boost_freq;
963 } else {
964 return intel_gpu_freq(rps, val: rps->boost_freq);
965 }
966}
967
968static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
969{
970 bool boost = false;
971
972 /* Validate against (static) hardware limits */
973 val = intel_freq_opcode(rps, val);
974 if (val < rps->min_freq || val > rps->max_freq)
975 return -EINVAL;
976
977 mutex_lock(lock: &rps->lock);
978 if (val != rps->boost_freq) {
979 rps->boost_freq = val;
980 boost = atomic_read(v: &rps->num_waiters);
981 }
982 mutex_unlock(lock: &rps->lock);
983 if (boost)
984 queue_work(wq: rps_to_gt(rps)->i915->unordered_wq, work: &rps->work);
985
986 return 0;
987}
988
989int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
990{
991 struct intel_guc_slpc *slpc;
992
993 if (rps_uses_slpc(rps)) {
994 slpc = rps_to_slpc(rps);
995
996 return intel_guc_slpc_set_boost_freq(slpc, val: freq);
997 } else {
998 return rps_set_boost_freq(rps, val: freq);
999 }
1000}
1001
1002void intel_rps_dec_waiters(struct intel_rps *rps)
1003{
1004 struct intel_guc_slpc *slpc;
1005
1006 if (rps_uses_slpc(rps)) {
1007 slpc = rps_to_slpc(rps);
1008
1009 /* Don't decrement num_waiters for req where increment was skipped */
1010 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING)
1011 return;
1012
1013 intel_guc_slpc_dec_waiters(slpc);
1014 } else {
1015 atomic_dec(v: &rps->num_waiters);
1016 }
1017}
1018
1019void intel_rps_boost(struct i915_request *rq)
1020{
1021 struct intel_guc_slpc *slpc;
1022
1023 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1024 return;
1025
1026 /* Waitboost is not needed for contexts marked with a Freq hint */
1027 if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags))
1028 return;
1029
1030 /* Serializes with i915_request_retire() */
1031 if (!test_and_set_bit(nr: I915_FENCE_FLAG_BOOST, addr: &rq->fence.flags)) {
1032 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1033
1034 if (rps_uses_slpc(rps)) {
1035 slpc = rps_to_slpc(rps);
1036
1037 /* Waitboost should not be done with power saving profile */
1038 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING)
1039 return;
1040
1041 /* Return if old value is non zero */
1042 if (!atomic_fetch_inc(v: &slpc->num_waiters)) {
1043 /*
1044 * Skip queuing boost work if frequency is already boosted,
1045 * but still increment num_waiters.
1046 */
1047 if (slpc->min_freq_softlimit >= slpc->boost_freq)
1048 return;
1049
1050 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1051 rq->fence.context, rq->fence.seqno);
1052 queue_work(wq: rps_to_gt(rps)->i915->unordered_wq,
1053 work: &slpc->boost_work);
1054 }
1055
1056 return;
1057 }
1058
1059 if (atomic_fetch_inc(v: &rps->num_waiters))
1060 return;
1061
1062 if (!intel_rps_is_active(rps))
1063 return;
1064
1065 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1066 rq->fence.context, rq->fence.seqno);
1067
1068 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1069 queue_work(wq: rps_to_gt(rps)->i915->unordered_wq, work: &rps->work);
1070
1071 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1072 }
1073}
1074
1075int intel_rps_set(struct intel_rps *rps, u8 val)
1076{
1077 int err;
1078
1079 lockdep_assert_held(&rps->lock);
1080 GEM_BUG_ON(val > rps->max_freq);
1081 GEM_BUG_ON(val < rps->min_freq);
1082
1083 if (intel_rps_is_active(rps)) {
1084 err = rps_set(rps, val, update: true);
1085 if (err)
1086 return err;
1087
1088 /*
1089 * Make sure we continue to get interrupts
1090 * until we hit the minimum or maximum frequencies.
1091 */
1092 if (intel_rps_has_interrupts(rps)) {
1093 struct intel_uncore *uncore = rps_to_uncore(rps);
1094
1095 set(uncore,
1096 GEN6_RP_INTERRUPT_LIMITS, val: rps_limits(rps, val));
1097
1098 set(uncore, GEN6_PMINTRMSK, val: rps_pm_mask(rps, val));
1099 }
1100 }
1101
1102 rps->cur_freq = val;
1103 return 0;
1104}
1105
1106static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1107{
1108 struct drm_i915_private *i915 = rps_to_i915(rps);
1109 struct intel_uncore *uncore = rps_to_uncore(rps);
1110
1111 if (IS_GEN9_LP(i915))
1112 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1113 else
1114 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1115}
1116
1117static void
1118mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1119{
1120 struct intel_uncore *uncore = rps_to_uncore(rps);
1121 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1122 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1123 intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1124 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1125 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1126 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1127
1128 /* MTL values are in units of 16.67 MHz */
1129 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1130 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1131 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1132}
1133
1134static void
1135__gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1136{
1137 struct drm_i915_private *i915 = rps_to_i915(rps);
1138 u32 rp_state_cap;
1139
1140 rp_state_cap = intel_rps_read_state_cap(rps);
1141
1142 /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1143 if (IS_GEN9_LP(i915)) {
1144 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1145 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1146 caps->min_freq = (rp_state_cap >> 0) & 0xff;
1147 } else {
1148 caps->rp0_freq = (rp_state_cap >> 0) & 0xff;
1149 if (GRAPHICS_VER(i915) >= 10)
1150 caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1151 intel_uncore_read(to_gt(i915)->uncore,
1152 GEN10_FREQ_INFO_REC));
1153 else
1154 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1155 caps->min_freq = (rp_state_cap >> 16) & 0xff;
1156 }
1157
1158 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1159 /*
1160 * In this case rp_state_cap register reports frequencies in
1161 * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1162 * units of 16.67 MHz
1163 */
1164 caps->rp0_freq *= GEN9_FREQ_SCALER;
1165 caps->rp1_freq *= GEN9_FREQ_SCALER;
1166 caps->min_freq *= GEN9_FREQ_SCALER;
1167 }
1168}
1169
1170/**
1171 * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1172 * @rps: the intel_rps structure
1173 * @caps: returned freq caps
1174 *
1175 * Returned "caps" frequencies should be converted to MHz using
1176 * intel_gpu_freq()
1177 */
1178void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1179{
1180 struct drm_i915_private *i915 = rps_to_i915(rps);
1181
1182 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1183 return mtl_get_freq_caps(rps, caps);
1184 else
1185 return __gen6_rps_get_freq_caps(rps, caps);
1186}
1187
1188static void gen6_rps_init(struct intel_rps *rps)
1189{
1190 struct drm_i915_private *i915 = rps_to_i915(rps);
1191 struct intel_rps_freq_caps caps;
1192
1193 gen6_rps_get_freq_caps(rps, caps: &caps);
1194 rps->rp0_freq = caps.rp0_freq;
1195 rps->rp1_freq = caps.rp1_freq;
1196 rps->min_freq = caps.min_freq;
1197
1198 /* hw_max = RP0 until we check for overclocking */
1199 rps->max_freq = rps->rp0_freq;
1200
1201 rps->efficient_freq = rps->rp1_freq;
1202 if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1203 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1204 u32 ddcc_status = 0;
1205 u32 mult = 1;
1206
1207 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1208 mult = GEN9_FREQ_SCALER;
1209 if (snb_pcode_read(uncore: rps_to_gt(rps)->uncore,
1210 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1211 val: &ddcc_status, NULL) == 0)
1212 rps->efficient_freq =
1213 clamp_t(u32,
1214 ((ddcc_status >> 8) & 0xff) * mult,
1215 rps->min_freq,
1216 rps->max_freq);
1217 }
1218}
1219
1220static bool rps_reset(struct intel_rps *rps)
1221{
1222 struct drm_i915_private *i915 = rps_to_i915(rps);
1223
1224 /* force a reset */
1225 rps->power.mode = -1;
1226 rps->last_freq = -1;
1227
1228 if (rps_set(rps, val: rps->min_freq, update: true)) {
1229 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1230 return false;
1231 }
1232
1233 rps->cur_freq = rps->min_freq;
1234 return true;
1235}
1236
1237/* See the Gen9_GT_PM_Programming_Guide doc for the below */
1238static bool gen9_rps_enable(struct intel_rps *rps)
1239{
1240 struct intel_gt *gt = rps_to_gt(rps);
1241 struct intel_uncore *uncore = gt->uncore;
1242
1243 /* Program defaults and thresholds for RPS */
1244 if (GRAPHICS_VER(gt->i915) == 9)
1245 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1246 GEN9_FREQUENCY(rps->rp1_freq));
1247
1248 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1249
1250 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1251
1252 return rps_reset(rps);
1253}
1254
1255static bool gen8_rps_enable(struct intel_rps *rps)
1256{
1257 struct intel_uncore *uncore = rps_to_uncore(rps);
1258
1259 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1260 HSW_FREQUENCY(rps->rp1_freq));
1261
1262 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1263
1264 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1265
1266 return rps_reset(rps);
1267}
1268
1269static bool gen6_rps_enable(struct intel_rps *rps)
1270{
1271 struct intel_uncore *uncore = rps_to_uncore(rps);
1272
1273 /* Power down if completely idle for over 50ms */
1274 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1275 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1276
1277 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1278 GEN6_PM_RP_DOWN_THRESHOLD |
1279 GEN6_PM_RP_DOWN_TIMEOUT);
1280
1281 return rps_reset(rps);
1282}
1283
1284static int chv_rps_max_freq(struct intel_rps *rps)
1285{
1286 struct drm_i915_private *i915 = rps_to_i915(rps);
1287 struct intel_gt *gt = rps_to_gt(rps);
1288 u32 val;
1289
1290 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE);
1291
1292 switch (gt->info.sseu.eu_total) {
1293 case 8:
1294 /* (2 * 4) config */
1295 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1296 break;
1297 case 12:
1298 /* (2 * 6) config */
1299 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1300 break;
1301 case 16:
1302 /* (2 * 8) config */
1303 default:
1304 /* Setting (2 * 8) Min RP0 for any other combination */
1305 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1306 break;
1307 }
1308
1309 return val & FB_GFX_FREQ_FUSE_MASK;
1310}
1311
1312static int chv_rps_rpe_freq(struct intel_rps *rps)
1313{
1314 struct drm_i915_private *i915 = rps_to_i915(rps);
1315 u32 val;
1316
1317 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_GPU_DUTYCYCLE_REG);
1318 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1319
1320 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1321}
1322
1323static int chv_rps_guar_freq(struct intel_rps *rps)
1324{
1325 struct drm_i915_private *i915 = rps_to_i915(rps);
1326 u32 val;
1327
1328 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE);
1329
1330 return val & FB_GFX_FREQ_FUSE_MASK;
1331}
1332
1333static u32 chv_rps_min_freq(struct intel_rps *rps)
1334{
1335 struct drm_i915_private *i915 = rps_to_i915(rps);
1336 u32 val;
1337
1338 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, FB_GFX_FMIN_AT_VMIN_FUSE);
1339 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1340
1341 return val & FB_GFX_FREQ_FUSE_MASK;
1342}
1343
1344static bool chv_rps_enable(struct intel_rps *rps)
1345{
1346 struct intel_uncore *uncore = rps_to_uncore(rps);
1347 struct drm_i915_private *i915 = rps_to_i915(rps);
1348 u32 val;
1349
1350 /* 1: Program defaults and thresholds for RPS*/
1351 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1352 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1353 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1354 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1355 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1356
1357 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1358
1359 /* 2: Enable RPS */
1360 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1361 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1362 GEN6_RP_MEDIA_IS_GFX |
1363 GEN6_RP_ENABLE |
1364 GEN6_RP_UP_BUSY_AVG |
1365 GEN6_RP_DOWN_IDLE_AVG);
1366
1367 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1368 GEN6_PM_RP_DOWN_THRESHOLD |
1369 GEN6_PM_RP_DOWN_TIMEOUT);
1370
1371 /* Setting Fixed Bias */
1372 vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1373
1374 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1375 vlv_iosf_sb_write(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val);
1376
1377 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
1378
1379 vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1380
1381 /* RPS code assumes GPLL is used */
1382 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1383 "GPLL not enabled\n");
1384
1385 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1386 str_yes_no(val & GPLLENABLE));
1387 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1388
1389 return rps_reset(rps);
1390}
1391
1392static int vlv_rps_guar_freq(struct intel_rps *rps)
1393{
1394 struct drm_i915_private *i915 = rps_to_i915(rps);
1395 u32 val, rp1;
1396
1397 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE);
1398
1399 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1400 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1401
1402 return rp1;
1403}
1404
1405static int vlv_rps_max_freq(struct intel_rps *rps)
1406{
1407 struct drm_i915_private *i915 = rps_to_i915(rps);
1408 u32 val, rp0;
1409
1410 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE);
1411
1412 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1413 /* Clamp to max */
1414 rp0 = min_t(u32, rp0, 0xea);
1415
1416 return rp0;
1417}
1418
1419static int vlv_rps_rpe_freq(struct intel_rps *rps)
1420{
1421 struct drm_i915_private *i915 = rps_to_i915(rps);
1422 u32 val, rpe;
1423
1424 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1425 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1426 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1427 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1428
1429 return rpe;
1430}
1431
1432static int vlv_rps_min_freq(struct intel_rps *rps)
1433{
1434 struct drm_i915_private *i915 = rps_to_i915(rps);
1435 u32 val;
1436
1437 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_LFM) & 0xff;
1438 /*
1439 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1440 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1441 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1442 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1443 * to make sure it matches what Punit accepts.
1444 */
1445 return max_t(u32, val, 0xc0);
1446}
1447
1448static bool vlv_rps_enable(struct intel_rps *rps)
1449{
1450 struct intel_uncore *uncore = rps_to_uncore(rps);
1451 struct drm_i915_private *i915 = rps_to_i915(rps);
1452 u32 val;
1453
1454 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1455 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1456 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1457 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1458 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1459
1460 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1461
1462 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1463 GEN6_RP_MEDIA_TURBO |
1464 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1465 GEN6_RP_MEDIA_IS_GFX |
1466 GEN6_RP_ENABLE |
1467 GEN6_RP_UP_BUSY_AVG |
1468 GEN6_RP_DOWN_IDLE_CONT);
1469
1470 /* WaGsvRC0ResidencyMethod:vlv */
1471 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1472
1473 vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1474
1475 /* Setting Fixed Bias */
1476 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1477 vlv_iosf_sb_write(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val);
1478
1479 val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
1480
1481 vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1482
1483 /* RPS code assumes GPLL is used */
1484 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1485 "GPLL not enabled\n");
1486
1487 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1488 str_yes_no(val & GPLLENABLE));
1489 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1490
1491 return rps_reset(rps);
1492}
1493
1494static unsigned long __ips_gfx_val(struct intel_ips *ips)
1495{
1496 struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1497 struct intel_uncore *uncore = rps_to_uncore(rps);
1498 unsigned int t, state1, state2;
1499 u32 pxvid, ext_v;
1500 u64 corr, corr2;
1501
1502 lockdep_assert_held(&mchdev_lock);
1503
1504 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1505 pxvid = (pxvid >> 24) & 0x7f;
1506 ext_v = pvid_to_extvid(i915: rps_to_i915(rps), pxvid);
1507
1508 state1 = ext_v;
1509
1510 /* Revel in the empirically derived constants */
1511
1512 /* Correction factor in 1/100000 units */
1513 t = ips_mch_val(uncore);
1514 if (t > 80)
1515 corr = t * 2349 + 135940;
1516 else if (t >= 50)
1517 corr = t * 964 + 29317;
1518 else /* < 50 */
1519 corr = t * 301 + 1004;
1520
1521 corr = div_u64(dividend: corr * 150142 * state1, divisor: 10000) - 78642;
1522 corr2 = div_u64(dividend: corr, divisor: 100000) * ips->corr;
1523
1524 state2 = div_u64(dividend: corr2 * state1, divisor: 10000);
1525 state2 /= 100; /* convert to mW */
1526
1527 __gen5_ips_update(ips);
1528
1529 return ips->gfx_power + state2;
1530}
1531
1532static bool has_busy_stats(struct intel_rps *rps)
1533{
1534 struct intel_engine_cs *engine;
1535 enum intel_engine_id id;
1536
1537 for_each_engine(engine, rps_to_gt(rps), id) {
1538 if (!intel_engine_supports_stats(engine))
1539 return false;
1540 }
1541
1542 return true;
1543}
1544
1545void intel_rps_enable(struct intel_rps *rps)
1546{
1547 struct drm_i915_private *i915 = rps_to_i915(rps);
1548 struct intel_uncore *uncore = rps_to_uncore(rps);
1549 bool enabled = false;
1550
1551 if (!HAS_RPS(i915))
1552 return;
1553
1554 if (rps_uses_slpc(rps))
1555 return;
1556
1557 intel_gt_check_clock_frequency(gt: rps_to_gt(rps));
1558
1559 intel_uncore_forcewake_get(uncore, domains: FORCEWAKE_ALL);
1560 if (rps->max_freq <= rps->min_freq)
1561 /* leave disabled, no room for dynamic reclocking */;
1562 else if (IS_CHERRYVIEW(i915))
1563 enabled = chv_rps_enable(rps);
1564 else if (IS_VALLEYVIEW(i915))
1565 enabled = vlv_rps_enable(rps);
1566 else if (GRAPHICS_VER(i915) >= 9)
1567 enabled = gen9_rps_enable(rps);
1568 else if (GRAPHICS_VER(i915) >= 8)
1569 enabled = gen8_rps_enable(rps);
1570 else if (GRAPHICS_VER(i915) >= 6)
1571 enabled = gen6_rps_enable(rps);
1572 else if (IS_IRONLAKE_M(i915))
1573 enabled = gen5_rps_enable(rps);
1574 else
1575 MISSING_CASE(GRAPHICS_VER(i915));
1576 intel_uncore_forcewake_put(uncore, domains: FORCEWAKE_ALL);
1577 if (!enabled)
1578 return;
1579
1580 GT_TRACE(rps_to_gt(rps),
1581 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n",
1582 rps->min_freq, rps->max_freq,
1583 intel_gpu_freq(rps, rps->min_freq),
1584 intel_gpu_freq(rps, rps->max_freq),
1585 rps->power.up_threshold,
1586 rps->power.down_threshold);
1587
1588 GEM_BUG_ON(rps->max_freq < rps->min_freq);
1589 GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1590
1591 GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1592 GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1593
1594 if (has_busy_stats(rps))
1595 intel_rps_set_timer(rps);
1596 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1597 intel_rps_set_interrupts(rps);
1598 else
1599 /* Ironlake currently uses intel_ips.ko */ {}
1600
1601 intel_rps_set_enabled(rps);
1602}
1603
1604static void gen6_rps_disable(struct intel_rps *rps)
1605{
1606 set(uncore: rps_to_uncore(rps), GEN6_RP_CONTROL, val: 0);
1607}
1608
1609void intel_rps_disable(struct intel_rps *rps)
1610{
1611 struct drm_i915_private *i915 = rps_to_i915(rps);
1612
1613 if (!intel_rps_is_enabled(rps))
1614 return;
1615
1616 intel_rps_clear_enabled(rps);
1617 intel_rps_clear_interrupts(rps);
1618 intel_rps_clear_timer(rps);
1619
1620 if (GRAPHICS_VER(i915) >= 6)
1621 gen6_rps_disable(rps);
1622 else if (IS_IRONLAKE_M(i915))
1623 gen5_rps_disable(rps);
1624}
1625
1626static int byt_gpu_freq(struct intel_rps *rps, int val)
1627{
1628 /*
1629 * N = val - 0xb7
1630 * Slow = Fast = GPLL ref * N
1631 */
1632 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1633}
1634
1635static int byt_freq_opcode(struct intel_rps *rps, int val)
1636{
1637 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1638}
1639
1640static int chv_gpu_freq(struct intel_rps *rps, int val)
1641{
1642 /*
1643 * N = val / 2
1644 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1645 */
1646 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1647}
1648
1649static int chv_freq_opcode(struct intel_rps *rps, int val)
1650{
1651 /* CHV needs even values */
1652 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1653}
1654
1655int intel_gpu_freq(struct intel_rps *rps, int val)
1656{
1657 struct drm_i915_private *i915 = rps_to_i915(rps);
1658
1659 if (GRAPHICS_VER(i915) >= 9)
1660 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1661 GEN9_FREQ_SCALER);
1662 else if (IS_CHERRYVIEW(i915))
1663 return chv_gpu_freq(rps, val);
1664 else if (IS_VALLEYVIEW(i915))
1665 return byt_gpu_freq(rps, val);
1666 else if (GRAPHICS_VER(i915) >= 6)
1667 return val * GT_FREQUENCY_MULTIPLIER;
1668 else
1669 return val;
1670}
1671
1672int intel_freq_opcode(struct intel_rps *rps, int val)
1673{
1674 struct drm_i915_private *i915 = rps_to_i915(rps);
1675
1676 if (GRAPHICS_VER(i915) >= 9)
1677 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1678 GT_FREQUENCY_MULTIPLIER);
1679 else if (IS_CHERRYVIEW(i915))
1680 return chv_freq_opcode(rps, val);
1681 else if (IS_VALLEYVIEW(i915))
1682 return byt_freq_opcode(rps, val);
1683 else if (GRAPHICS_VER(i915) >= 6)
1684 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1685 else
1686 return val;
1687}
1688
1689static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1690{
1691 struct drm_i915_private *i915 = rps_to_i915(rps);
1692
1693 rps->gpll_ref_freq =
1694 vlv_get_cck_clock(drm: &i915->drm, name: "GPLL ref",
1695 CCK_GPLL_CLOCK_CONTROL,
1696 ref_freq: i915->czclk_freq);
1697
1698 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1699 rps->gpll_ref_freq);
1700}
1701
1702static void vlv_rps_init(struct intel_rps *rps)
1703{
1704 struct drm_i915_private *i915 = rps_to_i915(rps);
1705
1706 vlv_iosf_sb_get(drm: &i915->drm,
1707 BIT(VLV_IOSF_SB_PUNIT) |
1708 BIT(VLV_IOSF_SB_NC) |
1709 BIT(VLV_IOSF_SB_CCK));
1710
1711 vlv_init_gpll_ref_freq(rps);
1712
1713 rps->max_freq = vlv_rps_max_freq(rps);
1714 rps->rp0_freq = rps->max_freq;
1715 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1716 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1717
1718 rps->efficient_freq = vlv_rps_rpe_freq(rps);
1719 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1720 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1721
1722 rps->rp1_freq = vlv_rps_guar_freq(rps);
1723 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1724 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1725
1726 rps->min_freq = vlv_rps_min_freq(rps);
1727 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1728 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1729
1730 vlv_iosf_sb_put(drm: &i915->drm,
1731 BIT(VLV_IOSF_SB_PUNIT) |
1732 BIT(VLV_IOSF_SB_NC) |
1733 BIT(VLV_IOSF_SB_CCK));
1734}
1735
1736static void chv_rps_init(struct intel_rps *rps)
1737{
1738 struct drm_i915_private *i915 = rps_to_i915(rps);
1739
1740 vlv_iosf_sb_get(drm: &i915->drm,
1741 BIT(VLV_IOSF_SB_PUNIT) |
1742 BIT(VLV_IOSF_SB_NC) |
1743 BIT(VLV_IOSF_SB_CCK));
1744
1745 vlv_init_gpll_ref_freq(rps);
1746
1747 rps->max_freq = chv_rps_max_freq(rps);
1748 rps->rp0_freq = rps->max_freq;
1749 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1750 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1751
1752 rps->efficient_freq = chv_rps_rpe_freq(rps);
1753 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1754 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1755
1756 rps->rp1_freq = chv_rps_guar_freq(rps);
1757 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1758 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1759
1760 rps->min_freq = chv_rps_min_freq(rps);
1761 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1762 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1763
1764 vlv_iosf_sb_put(drm: &i915->drm,
1765 BIT(VLV_IOSF_SB_PUNIT) |
1766 BIT(VLV_IOSF_SB_NC) |
1767 BIT(VLV_IOSF_SB_CCK));
1768
1769 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1770 rps->rp1_freq | rps->min_freq) & 1,
1771 "Odd GPU freq values\n");
1772}
1773
1774static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1775{
1776 ei->ktime = ktime_get_raw();
1777 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1778 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1779}
1780
1781static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1782{
1783 struct intel_uncore *uncore = rps_to_uncore(rps);
1784 const struct intel_rps_ei *prev = &rps->ei;
1785 struct intel_rps_ei now;
1786 u32 events = 0;
1787
1788 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1789 return 0;
1790
1791 vlv_c0_read(uncore, ei: &now);
1792
1793 if (prev->ktime) {
1794 u64 time, c0;
1795 u32 render, media;
1796
1797 time = ktime_us_delta(later: now.ktime, earlier: prev->ktime);
1798
1799 time *= rps_to_i915(rps)->czclk_freq;
1800
1801 /* Workload can be split between render + media,
1802 * e.g. SwapBuffers being blitted in X after being rendered in
1803 * mesa. To account for this we need to combine both engines
1804 * into our activity counter.
1805 */
1806 render = now.render_c0 - prev->render_c0;
1807 media = now.media_c0 - prev->media_c0;
1808 c0 = max(render, media);
1809 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1810
1811 if (c0 > time * rps->power.up_threshold)
1812 events = GEN6_PM_RP_UP_THRESHOLD;
1813 else if (c0 < time * rps->power.down_threshold)
1814 events = GEN6_PM_RP_DOWN_THRESHOLD;
1815 }
1816
1817 rps->ei = now;
1818 return events;
1819}
1820
1821static void rps_work(struct work_struct *work)
1822{
1823 struct intel_rps *rps = container_of(work, typeof(*rps), work);
1824 struct intel_gt *gt = rps_to_gt(rps);
1825 struct drm_i915_private *i915 = rps_to_i915(rps);
1826 bool client_boost = false;
1827 int new_freq, adj, min, max;
1828 u32 pm_iir = 0;
1829
1830 spin_lock_irq(lock: gt->irq_lock);
1831 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1832 client_boost = atomic_read(v: &rps->num_waiters);
1833 spin_unlock_irq(lock: gt->irq_lock);
1834
1835 /* Make sure we didn't queue anything we're not going to process. */
1836 if (!pm_iir && !client_boost)
1837 goto out;
1838
1839 mutex_lock(lock: &rps->lock);
1840 if (!intel_rps_is_active(rps)) {
1841 mutex_unlock(lock: &rps->lock);
1842 return;
1843 }
1844
1845 pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1846
1847 adj = rps->last_adj;
1848 new_freq = rps->cur_freq;
1849 min = rps->min_freq_softlimit;
1850 max = rps->max_freq_softlimit;
1851 if (client_boost)
1852 max = rps->max_freq;
1853
1854 GT_TRACE(gt,
1855 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1856 pm_iir, str_yes_no(client_boost),
1857 adj, new_freq, min, max);
1858
1859 if (client_boost && new_freq < rps->boost_freq) {
1860 new_freq = rps->boost_freq;
1861 adj = 0;
1862 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1863 if (adj > 0)
1864 adj *= 2;
1865 else /* CHV needs even encode values */
1866 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1867
1868 if (new_freq >= rps->max_freq_softlimit)
1869 adj = 0;
1870 } else if (client_boost) {
1871 adj = 0;
1872 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1873 if (rps->cur_freq > rps->efficient_freq)
1874 new_freq = rps->efficient_freq;
1875 else if (rps->cur_freq > rps->min_freq_softlimit)
1876 new_freq = rps->min_freq_softlimit;
1877 adj = 0;
1878 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1879 if (adj < 0)
1880 adj *= 2;
1881 else /* CHV needs even encode values */
1882 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1883
1884 if (new_freq <= rps->min_freq_softlimit)
1885 adj = 0;
1886 } else { /* unknown event */
1887 adj = 0;
1888 }
1889
1890 /*
1891 * sysfs frequency limits may have snuck in while
1892 * servicing the interrupt
1893 */
1894 new_freq += adj;
1895 new_freq = clamp_t(int, new_freq, min, max);
1896
1897 if (intel_rps_set(rps, val: new_freq)) {
1898 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1899 adj = 0;
1900 }
1901 rps->last_adj = adj;
1902
1903 mutex_unlock(lock: &rps->lock);
1904
1905out:
1906 spin_lock_irq(lock: gt->irq_lock);
1907 gen6_gt_pm_unmask_irq(gt, mask: rps->pm_events);
1908 spin_unlock_irq(lock: gt->irq_lock);
1909}
1910
1911void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1912{
1913 struct intel_gt *gt = rps_to_gt(rps);
1914 const u32 events = rps->pm_events & pm_iir;
1915
1916 lockdep_assert_held(gt->irq_lock);
1917
1918 if (unlikely(!events))
1919 return;
1920
1921 GT_TRACE(gt, "irq events:%x\n", events);
1922
1923 gen6_gt_pm_mask_irq(gt, mask: events);
1924
1925 rps->pm_iir |= events;
1926 queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
1927}
1928
1929void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1930{
1931 struct intel_gt *gt = rps_to_gt(rps);
1932 u32 events;
1933
1934 events = pm_iir & rps->pm_events;
1935 if (events) {
1936 spin_lock(lock: gt->irq_lock);
1937
1938 GT_TRACE(gt, "irq events:%x\n", events);
1939
1940 gen6_gt_pm_mask_irq(gt, mask: events);
1941 rps->pm_iir |= events;
1942
1943 queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
1944 spin_unlock(lock: gt->irq_lock);
1945 }
1946
1947 if (GRAPHICS_VER(gt->i915) >= 8)
1948 return;
1949
1950 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1951 intel_engine_cs_irq(engine: gt->engine[VECS0], iir: pm_iir >> 10);
1952
1953 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1954 drm_dbg(&rps_to_i915(rps)->drm,
1955 "Command parser error, pm_iir 0x%08x\n", pm_iir);
1956}
1957
1958void gen5_rps_irq_handler(struct intel_rps *rps)
1959{
1960 struct intel_uncore *uncore = rps_to_uncore(rps);
1961 u32 busy_up, busy_down, max_avg, min_avg;
1962 u8 new_freq;
1963
1964 spin_lock(lock: &mchdev_lock);
1965
1966 intel_uncore_write16(uncore,
1967 MEMINTRSTS,
1968 val: intel_uncore_read(uncore, MEMINTRSTS));
1969
1970 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1971 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1972 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1973 max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1974 min_avg = intel_uncore_read(uncore, RCBMINAVG);
1975
1976 /* Handle RCS change request from hw */
1977 new_freq = rps->cur_freq;
1978 if (busy_up > max_avg)
1979 new_freq++;
1980 else if (busy_down < min_avg)
1981 new_freq--;
1982 new_freq = clamp(new_freq,
1983 rps->min_freq_softlimit,
1984 rps->max_freq_softlimit);
1985
1986 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, val: new_freq))
1987 rps->cur_freq = new_freq;
1988
1989 spin_unlock(lock: &mchdev_lock);
1990}
1991
1992void intel_rps_init_early(struct intel_rps *rps)
1993{
1994 mutex_init(&rps->lock);
1995 mutex_init(&rps->power.mutex);
1996
1997 INIT_WORK(&rps->work, rps_work);
1998 timer_setup(&rps->timer, rps_timer, 0);
1999
2000 atomic_set(v: &rps->num_waiters, i: 0);
2001}
2002
2003void intel_rps_init(struct intel_rps *rps)
2004{
2005 struct drm_i915_private *i915 = rps_to_i915(rps);
2006
2007 if (rps_uses_slpc(rps))
2008 return;
2009
2010 if (IS_CHERRYVIEW(i915))
2011 chv_rps_init(rps);
2012 else if (IS_VALLEYVIEW(i915))
2013 vlv_rps_init(rps);
2014 else if (GRAPHICS_VER(i915) >= 6)
2015 gen6_rps_init(rps);
2016 else if (IS_IRONLAKE_M(i915))
2017 gen5_rps_init(rps);
2018
2019 /* Derive initial user preferences/limits from the hardware limits */
2020 rps->max_freq_softlimit = rps->max_freq;
2021 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
2022 rps->min_freq_softlimit = rps->min_freq;
2023 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
2024
2025 /* After setting max-softlimit, find the overclock max freq */
2026 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
2027 u32 params = 0;
2028
2029 snb_pcode_read(uncore: rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, val: &params, NULL);
2030 if (params & BIT(31)) { /* OC supported */
2031 drm_dbg(&i915->drm,
2032 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2033 (rps->max_freq & 0xff) * 50,
2034 (params & 0xff) * 50);
2035 rps->max_freq = params & 0xff;
2036 }
2037 }
2038
2039 /* Set default thresholds in % */
2040 rps->power.up_threshold = 95;
2041 rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold;
2042 rps->power.down_threshold = 85;
2043 rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold;
2044
2045 /* Finally allow us to boost to max by default */
2046 rps->boost_freq = rps->max_freq;
2047 rps->idle_freq = rps->min_freq;
2048
2049 /* Start in the middle, from here we will autotune based on workload */
2050 rps->cur_freq = rps->efficient_freq;
2051
2052 rps->pm_intrmsk_mbz = 0;
2053
2054 /*
2055 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2056 * if GEN6_PM_UP_EI_EXPIRED is masked.
2057 *
2058 * TODO: verify if this can be reproduced on VLV,CHV.
2059 */
2060 if (GRAPHICS_VER(i915) <= 7)
2061 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2062
2063 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2064 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2065
2066 /* GuC needs ARAT expired interrupt unmasked */
2067 if (intel_uc_uses_guc_submission(uc: &rps_to_gt(rps)->uc))
2068 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2069}
2070
2071void intel_rps_sanitize(struct intel_rps *rps)
2072{
2073 if (rps_uses_slpc(rps))
2074 return;
2075
2076 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2077 rps_disable_interrupts(rps);
2078}
2079
2080u32 intel_rps_read_rpstat(struct intel_rps *rps)
2081{
2082 struct drm_i915_private *i915 = rps_to_i915(rps);
2083 i915_reg_t rpstat;
2084
2085 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2086
2087 return intel_uncore_read(uncore: rps_to_gt(rps)->uncore, reg: rpstat);
2088}
2089
2090static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2091{
2092 struct drm_i915_private *i915 = rps_to_i915(rps);
2093 u32 cagf;
2094
2095 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2096 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2097 else if (GRAPHICS_VER(i915) >= 12)
2098 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2099 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2100 cagf = REG_FIELD_GET(RPE_MASK, rpstat);
2101 else if (GRAPHICS_VER(i915) >= 9)
2102 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
2103 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2104 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
2105 else if (GRAPHICS_VER(i915) >= 6)
2106 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
2107 else
2108 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
2109
2110 return cagf;
2111}
2112
2113static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
2114{
2115 struct drm_i915_private *i915 = rps_to_i915(rps);
2116 struct intel_uncore *uncore = rps_to_uncore(rps);
2117 i915_reg_t r = INVALID_MMIO_REG;
2118 u32 freq;
2119
2120 /*
2121 * For Gen12+ reading freq from HW does not need a forcewake and
2122 * registers will return 0 freq when GT is in RC6
2123 */
2124 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2125 r = MTL_MIRROR_TARGET_WP1;
2126 } else if (GRAPHICS_VER(i915) >= 12) {
2127 r = GEN12_RPSTAT1;
2128 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2129 vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
2130 freq = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
2131 vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
2132 } else if (GRAPHICS_VER(i915) >= 6) {
2133 r = GEN6_RPSTAT1;
2134 } else {
2135 r = MEMSTAT_ILK;
2136 }
2137
2138 if (i915_mmio_reg_valid(r))
2139 freq = take_fw ? intel_uncore_read(uncore, reg: r) : intel_uncore_read_fw(uncore, r);
2140
2141 return intel_rps_get_cagf(rps, rpstat: freq);
2142}
2143
2144static u32 read_cagf(struct intel_rps *rps)
2145{
2146 return __read_cagf(rps, take_fw: true);
2147}
2148
2149u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2150{
2151 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2152 intel_wakeref_t wakeref;
2153 u32 freq = 0;
2154
2155 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2156 freq = intel_gpu_freq(rps, val: read_cagf(rps));
2157
2158 return freq;
2159}
2160
2161u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2162{
2163 return intel_gpu_freq(rps, val: __read_cagf(rps, take_fw: false));
2164}
2165
2166static u32 intel_rps_read_punit_req(struct intel_rps *rps)
2167{
2168 struct intel_uncore *uncore = rps_to_uncore(rps);
2169 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2170 intel_wakeref_t wakeref;
2171 u32 freq = 0;
2172
2173 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2174 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2175
2176 return freq;
2177}
2178
2179static u32 intel_rps_get_req(u32 pureq)
2180{
2181 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2182
2183 return req;
2184}
2185
2186u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2187{
2188 u32 freq = intel_rps_get_req(pureq: intel_rps_read_punit_req(rps));
2189
2190 return intel_gpu_freq(rps, val: freq);
2191}
2192
2193u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2194{
2195 if (rps_uses_slpc(rps))
2196 return intel_rps_read_punit_req_frequency(rps);
2197 else
2198 return intel_gpu_freq(rps, val: rps->cur_freq);
2199}
2200
2201u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2202{
2203 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2204
2205 if (rps_uses_slpc(rps))
2206 return slpc->max_freq_softlimit;
2207 else
2208 return intel_gpu_freq(rps, val: rps->max_freq_softlimit);
2209}
2210
2211/**
2212 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2213 * @rps: the intel_rps structure
2214 *
2215 * Returns the max frequency in a raw format. In newer platforms raw is in
2216 * units of 50 MHz.
2217 */
2218u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2219{
2220 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2221 u32 freq;
2222
2223 if (rps_uses_slpc(rps)) {
2224 return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2225 GT_FREQUENCY_MULTIPLIER);
2226 } else {
2227 freq = rps->max_freq;
2228 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2229 /* Convert GT frequency to 50 MHz units */
2230 freq /= GEN9_FREQ_SCALER;
2231 }
2232 return freq;
2233 }
2234}
2235
2236u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2237{
2238 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2239
2240 if (rps_uses_slpc(rps))
2241 return slpc->rp0_freq;
2242 else
2243 return intel_gpu_freq(rps, val: rps->rp0_freq);
2244}
2245
2246u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2247{
2248 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2249
2250 if (rps_uses_slpc(rps))
2251 return slpc->rp1_freq;
2252 else
2253 return intel_gpu_freq(rps, val: rps->rp1_freq);
2254}
2255
2256u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2257{
2258 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2259
2260 if (rps_uses_slpc(rps))
2261 return slpc->min_freq;
2262 else
2263 return intel_gpu_freq(rps, val: rps->min_freq);
2264}
2265
2266static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2267{
2268 struct intel_gt *gt = rps_to_gt(rps);
2269 struct drm_i915_private *i915 = gt->i915;
2270 struct intel_uncore *uncore = gt->uncore;
2271 struct intel_rps_freq_caps caps;
2272 u32 rp_state_limits;
2273 u32 gt_perf_status;
2274 u32 rpmodectl, rpinclimit, rpdeclimit;
2275 u32 rpstat, cagf, reqf;
2276 u32 rpcurupei, rpcurup, rpprevup;
2277 u32 rpcurdownei, rpcurdown, rpprevdown;
2278 u32 rpupei, rpupt, rpdownei, rpdownt;
2279 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2280
2281 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2282 gen6_rps_get_freq_caps(rps, caps: &caps);
2283 if (IS_GEN9_LP(i915))
2284 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2285 else
2286 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2287
2288 /* RPSTAT1 is in the GT power well */
2289 intel_uncore_forcewake_get(uncore, domains: FORCEWAKE_ALL);
2290
2291 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2292 if (GRAPHICS_VER(i915) >= 9) {
2293 reqf >>= 23;
2294 } else {
2295 reqf &= ~GEN6_TURBO_DISABLE;
2296 if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2297 reqf >>= 24;
2298 else
2299 reqf >>= 25;
2300 }
2301 reqf = intel_gpu_freq(rps, val: reqf);
2302
2303 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2304 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2305 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2306
2307 rpstat = intel_rps_read_rpstat(rps);
2308 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2309 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2310 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2311 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2312 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2313 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2314
2315 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2316 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2317
2318 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2319 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2320
2321 cagf = intel_rps_read_actual_frequency(rps);
2322
2323 intel_uncore_forcewake_put(uncore, domains: FORCEWAKE_ALL);
2324
2325 if (GRAPHICS_VER(i915) >= 11) {
2326 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2327 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2328 /*
2329 * The equivalent to the PM ISR & IIR cannot be read
2330 * without affecting the current state of the system
2331 */
2332 pm_isr = 0;
2333 pm_iir = 0;
2334 } else if (GRAPHICS_VER(i915) >= 8) {
2335 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2336 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2337 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2338 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2339 } else {
2340 pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2341 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2342 pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2343 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2344 }
2345 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2346
2347 drm_printf(p, f: "Video Turbo Mode: %s\n",
2348 str_yes_no(v: rpmodectl & GEN6_RP_MEDIA_TURBO));
2349 drm_printf(p, f: "HW control enabled: %s\n",
2350 str_yes_no(v: rpmodectl & GEN6_RP_ENABLE));
2351 drm_printf(p, f: "SW control enabled: %s\n",
2352 str_yes_no(v: (rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2353
2354 drm_printf(p, f: "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2355 pm_ier, pm_imr, pm_mask);
2356 if (GRAPHICS_VER(i915) <= 10)
2357 drm_printf(p, f: "PM ISR=0x%08x IIR=0x%08x\n",
2358 pm_isr, pm_iir);
2359 drm_printf(p, f: "pm_intrmsk_mbz: 0x%08x\n",
2360 rps->pm_intrmsk_mbz);
2361 drm_printf(p, f: "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2362 drm_printf(p, f: "Render p-state ratio: %d\n",
2363 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2364 drm_printf(p, f: "Render p-state VID: %d\n",
2365 gt_perf_status & 0xff);
2366 drm_printf(p, f: "Render p-state limit: %d\n",
2367 rp_state_limits & 0xff);
2368 drm_printf(p, f: "RPSTAT1: 0x%08x\n", rpstat);
2369 drm_printf(p, f: "RPMODECTL: 0x%08x\n", rpmodectl);
2370 drm_printf(p, f: "RPINCLIMIT: 0x%08x\n", rpinclimit);
2371 drm_printf(p, f: "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2372 drm_printf(p, f: "RPNSWREQ: %dMHz\n", reqf);
2373 drm_printf(p, f: "CAGF: %dMHz\n", cagf);
2374 drm_printf(p, f: "RP CUR UP EI: %d (%lldns)\n",
2375 rpcurupei,
2376 intel_gt_pm_interval_to_ns(gt, count: rpcurupei));
2377 drm_printf(p, f: "RP CUR UP: %d (%lldns)\n",
2378 rpcurup, intel_gt_pm_interval_to_ns(gt, count: rpcurup));
2379 drm_printf(p, f: "RP PREV UP: %d (%lldns)\n",
2380 rpprevup, intel_gt_pm_interval_to_ns(gt, count: rpprevup));
2381 drm_printf(p, f: "Up threshold: %d%%\n",
2382 rps->power.up_threshold);
2383 drm_printf(p, f: "RP UP EI: %d (%lldns)\n",
2384 rpupei, intel_gt_pm_interval_to_ns(gt, count: rpupei));
2385 drm_printf(p, f: "RP UP THRESHOLD: %d (%lldns)\n",
2386 rpupt, intel_gt_pm_interval_to_ns(gt, count: rpupt));
2387
2388 drm_printf(p, f: "RP CUR DOWN EI: %d (%lldns)\n",
2389 rpcurdownei,
2390 intel_gt_pm_interval_to_ns(gt, count: rpcurdownei));
2391 drm_printf(p, f: "RP CUR DOWN: %d (%lldns)\n",
2392 rpcurdown,
2393 intel_gt_pm_interval_to_ns(gt, count: rpcurdown));
2394 drm_printf(p, f: "RP PREV DOWN: %d (%lldns)\n",
2395 rpprevdown,
2396 intel_gt_pm_interval_to_ns(gt, count: rpprevdown));
2397 drm_printf(p, f: "Down threshold: %d%%\n",
2398 rps->power.down_threshold);
2399 drm_printf(p, f: "RP DOWN EI: %d (%lldns)\n",
2400 rpdownei, intel_gt_pm_interval_to_ns(gt, count: rpdownei));
2401 drm_printf(p, f: "RP DOWN THRESHOLD: %d (%lldns)\n",
2402 rpdownt, intel_gt_pm_interval_to_ns(gt, count: rpdownt));
2403
2404 drm_printf(p, f: "Lowest (RPN) frequency: %dMHz\n",
2405 intel_gpu_freq(rps, val: caps.min_freq));
2406 drm_printf(p, f: "Nominal (RP1) frequency: %dMHz\n",
2407 intel_gpu_freq(rps, val: caps.rp1_freq));
2408 drm_printf(p, f: "Max non-overclocked (RP0) frequency: %dMHz\n",
2409 intel_gpu_freq(rps, val: caps.rp0_freq));
2410 drm_printf(p, f: "Max overclocked frequency: %dMHz\n",
2411 intel_gpu_freq(rps, val: rps->max_freq));
2412
2413 drm_printf(p, f: "Current freq: %d MHz\n",
2414 intel_gpu_freq(rps, val: rps->cur_freq));
2415 drm_printf(p, f: "Actual freq: %d MHz\n", cagf);
2416 drm_printf(p, f: "Idle freq: %d MHz\n",
2417 intel_gpu_freq(rps, val: rps->idle_freq));
2418 drm_printf(p, f: "Min freq: %d MHz\n",
2419 intel_gpu_freq(rps, val: rps->min_freq));
2420 drm_printf(p, f: "Boost freq: %d MHz\n",
2421 intel_gpu_freq(rps, val: rps->boost_freq));
2422 drm_printf(p, f: "Max freq: %d MHz\n",
2423 intel_gpu_freq(rps, val: rps->max_freq));
2424 drm_printf(p,
2425 f: "efficient (RPe) frequency: %d MHz\n",
2426 intel_gpu_freq(rps, val: rps->efficient_freq));
2427}
2428
2429static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2430{
2431 struct intel_gt *gt = rps_to_gt(rps);
2432 struct intel_uncore *uncore = gt->uncore;
2433 struct intel_rps_freq_caps caps;
2434 u32 pm_mask;
2435
2436 gen6_rps_get_freq_caps(rps, caps: &caps);
2437 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2438
2439 drm_printf(p, f: "PM MASK=0x%08x\n", pm_mask);
2440 drm_printf(p, f: "pm_intrmsk_mbz: 0x%08x\n",
2441 rps->pm_intrmsk_mbz);
2442 drm_printf(p, f: "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
2443 drm_printf(p, f: "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2444 drm_printf(p, f: "Lowest (RPN) frequency: %dMHz\n",
2445 intel_gpu_freq(rps, val: caps.min_freq));
2446 drm_printf(p, f: "Nominal (RP1) frequency: %dMHz\n",
2447 intel_gpu_freq(rps, val: caps.rp1_freq));
2448 drm_printf(p, f: "Max non-overclocked (RP0) frequency: %dMHz\n",
2449 intel_gpu_freq(rps, val: caps.rp0_freq));
2450 drm_printf(p, f: "Current freq: %d MHz\n",
2451 intel_rps_get_requested_frequency(rps));
2452 drm_printf(p, f: "Actual freq: %d MHz\n",
2453 intel_rps_read_actual_frequency(rps));
2454 drm_printf(p, f: "Min freq: %d MHz\n",
2455 intel_rps_get_min_frequency(rps));
2456 drm_printf(p, f: "Boost freq: %d MHz\n",
2457 intel_rps_get_boost_frequency(rps));
2458 drm_printf(p, f: "Max freq: %d MHz\n",
2459 intel_rps_get_max_frequency(rps));
2460 drm_printf(p,
2461 f: "efficient (RPe) frequency: %d MHz\n",
2462 intel_gpu_freq(rps, val: caps.rp1_freq));
2463}
2464
2465void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2466{
2467 if (rps_uses_slpc(rps))
2468 return slpc_frequency_dump(rps, p);
2469 else
2470 return rps_frequency_dump(rps, p);
2471}
2472
2473static int set_max_freq(struct intel_rps *rps, u32 val)
2474{
2475 struct drm_i915_private *i915 = rps_to_i915(rps);
2476 int ret = 0;
2477
2478 mutex_lock(lock: &rps->lock);
2479
2480 val = intel_freq_opcode(rps, val);
2481 if (val < rps->min_freq ||
2482 val > rps->max_freq ||
2483 val < rps->min_freq_softlimit) {
2484 ret = -EINVAL;
2485 goto unlock;
2486 }
2487
2488 if (val > rps->rp0_freq)
2489 drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2490 intel_gpu_freq(rps, val));
2491
2492 rps->max_freq_softlimit = val;
2493
2494 val = clamp_t(int, rps->cur_freq,
2495 rps->min_freq_softlimit,
2496 rps->max_freq_softlimit);
2497
2498 /*
2499 * We still need *_set_rps to process the new max_delay and
2500 * update the interrupt limits and PMINTRMSK even though
2501 * frequency request may be unchanged.
2502 */
2503 intel_rps_set(rps, val);
2504
2505unlock:
2506 mutex_unlock(lock: &rps->lock);
2507
2508 return ret;
2509}
2510
2511int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2512{
2513 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2514
2515 if (rps_uses_slpc(rps))
2516 return intel_guc_slpc_set_max_freq(slpc, val);
2517 else
2518 return set_max_freq(rps, val);
2519}
2520
2521u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2522{
2523 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2524
2525 if (rps_uses_slpc(rps))
2526 return slpc->min_freq_softlimit;
2527 else
2528 return intel_gpu_freq(rps, val: rps->min_freq_softlimit);
2529}
2530
2531/**
2532 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2533 * @rps: the intel_rps structure
2534 *
2535 * Returns the min frequency in a raw format. In newer platforms raw is in
2536 * units of 50 MHz.
2537 */
2538u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2539{
2540 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2541 u32 freq;
2542
2543 if (rps_uses_slpc(rps)) {
2544 return DIV_ROUND_CLOSEST(slpc->min_freq,
2545 GT_FREQUENCY_MULTIPLIER);
2546 } else {
2547 freq = rps->min_freq;
2548 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2549 /* Convert GT frequency to 50 MHz units */
2550 freq /= GEN9_FREQ_SCALER;
2551 }
2552 return freq;
2553 }
2554}
2555
2556static int set_min_freq(struct intel_rps *rps, u32 val)
2557{
2558 int ret = 0;
2559
2560 mutex_lock(lock: &rps->lock);
2561
2562 val = intel_freq_opcode(rps, val);
2563 if (val < rps->min_freq ||
2564 val > rps->max_freq ||
2565 val > rps->max_freq_softlimit) {
2566 ret = -EINVAL;
2567 goto unlock;
2568 }
2569
2570 rps->min_freq_softlimit = val;
2571
2572 val = clamp_t(int, rps->cur_freq,
2573 rps->min_freq_softlimit,
2574 rps->max_freq_softlimit);
2575
2576 /*
2577 * We still need *_set_rps to process the new min_delay and
2578 * update the interrupt limits and PMINTRMSK even though
2579 * frequency request may be unchanged.
2580 */
2581 intel_rps_set(rps, val);
2582
2583unlock:
2584 mutex_unlock(lock: &rps->lock);
2585
2586 return ret;
2587}
2588
2589int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2590{
2591 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2592
2593 if (rps_uses_slpc(rps))
2594 return intel_guc_slpc_set_min_freq(slpc, val);
2595 else
2596 return set_min_freq(rps, val);
2597}
2598
2599u8 intel_rps_get_up_threshold(struct intel_rps *rps)
2600{
2601 return rps->power.up_threshold;
2602}
2603
2604static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)
2605{
2606 int ret;
2607
2608 if (val > 100)
2609 return -EINVAL;
2610
2611 ret = mutex_lock_interruptible(lock: &rps->lock);
2612 if (ret)
2613 return ret;
2614
2615 if (*threshold == val)
2616 goto out_unlock;
2617
2618 *threshold = val;
2619
2620 /* Force reset. */
2621 rps->last_freq = -1;
2622 mutex_lock(lock: &rps->power.mutex);
2623 rps->power.mode = -1;
2624 mutex_unlock(lock: &rps->power.mutex);
2625
2626 intel_rps_set(rps, clamp(rps->cur_freq,
2627 rps->min_freq_softlimit,
2628 rps->max_freq_softlimit));
2629
2630out_unlock:
2631 mutex_unlock(lock: &rps->lock);
2632
2633 return ret;
2634}
2635
2636int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)
2637{
2638 return rps_set_threshold(rps, threshold: &rps->power.up_threshold, val: threshold);
2639}
2640
2641u8 intel_rps_get_down_threshold(struct intel_rps *rps)
2642{
2643 return rps->power.down_threshold;
2644}
2645
2646int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)
2647{
2648 return rps_set_threshold(rps, threshold: &rps->power.down_threshold, val: threshold);
2649}
2650
2651static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2652{
2653 struct intel_uncore *uncore = rps_to_uncore(rps);
2654 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2655
2656 /* Allow punit to process software requests */
2657 intel_uncore_write(uncore, GEN6_RP_CONTROL, val: state);
2658}
2659
2660void intel_rps_raise_unslice(struct intel_rps *rps)
2661{
2662 struct intel_uncore *uncore = rps_to_uncore(rps);
2663
2664 mutex_lock(lock: &rps->lock);
2665
2666 if (rps_uses_slpc(rps)) {
2667 /* RP limits have not been initialized yet for SLPC path */
2668 struct intel_rps_freq_caps caps;
2669
2670 gen6_rps_get_freq_caps(rps, caps: &caps);
2671
2672 intel_rps_set_manual(rps, enable: true);
2673 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2674 val: ((caps.rp0_freq <<
2675 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2676 GEN9_IGNORE_SLICE_RATIO));
2677 intel_rps_set_manual(rps, enable: false);
2678 } else {
2679 intel_rps_set(rps, val: rps->rp0_freq);
2680 }
2681
2682 mutex_unlock(lock: &rps->lock);
2683}
2684
2685void intel_rps_lower_unslice(struct intel_rps *rps)
2686{
2687 struct intel_uncore *uncore = rps_to_uncore(rps);
2688
2689 mutex_lock(lock: &rps->lock);
2690
2691 if (rps_uses_slpc(rps)) {
2692 /* RP limits have not been initialized yet for SLPC path */
2693 struct intel_rps_freq_caps caps;
2694
2695 gen6_rps_get_freq_caps(rps, caps: &caps);
2696
2697 intel_rps_set_manual(rps, enable: true);
2698 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2699 val: ((caps.min_freq <<
2700 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2701 GEN9_IGNORE_SLICE_RATIO));
2702 intel_rps_set_manual(rps, enable: false);
2703 } else {
2704 intel_rps_set(rps, val: rps->min_freq);
2705 }
2706
2707 mutex_unlock(lock: &rps->lock);
2708}
2709
2710static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2711{
2712 struct intel_gt *gt = rps_to_gt(rps);
2713 intel_wakeref_t wakeref;
2714 u32 val;
2715
2716 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2717 val = intel_uncore_read(uncore: gt->uncore, reg: reg32);
2718
2719 return val;
2720}
2721
2722bool rps_read_mask_mmio(struct intel_rps *rps,
2723 i915_reg_t reg32, u32 mask)
2724{
2725 return rps_read_mmio(rps, reg32) & mask;
2726}
2727
2728/* External interface for intel_ips.ko */
2729
2730static struct drm_i915_private __rcu *ips_mchdev;
2731
2732/*
2733 * Tells the intel_ips driver that the i915 driver is now loaded, if
2734 * IPS got loaded first.
2735 *
2736 * This awkward dance is so that neither module has to depend on the
2737 * other in order for IPS to do the appropriate communication of
2738 * GPU turbo limits to i915.
2739 */
2740static void
2741ips_ping_for_i915_load(void)
2742{
2743 void (*link)(void);
2744
2745 link = symbol_get(ips_link_to_i915_driver);
2746 if (link) {
2747 link();
2748 symbol_put(ips_link_to_i915_driver);
2749 }
2750}
2751
2752void intel_rps_driver_register(struct intel_rps *rps)
2753{
2754 struct intel_gt *gt = rps_to_gt(rps);
2755
2756 /*
2757 * We only register the i915 ips part with intel-ips once everything is
2758 * set up, to avoid intel-ips sneaking in and reading bogus values.
2759 */
2760 if (GRAPHICS_VER(gt->i915) == 5) {
2761 GEM_BUG_ON(ips_mchdev);
2762 rcu_assign_pointer(ips_mchdev, gt->i915);
2763 ips_ping_for_i915_load();
2764 }
2765}
2766
2767void intel_rps_driver_unregister(struct intel_rps *rps)
2768{
2769 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2770 rcu_assign_pointer(ips_mchdev, NULL);
2771}
2772
2773static struct drm_i915_private *mchdev_get(void)
2774{
2775 struct drm_i915_private *i915;
2776
2777 rcu_read_lock();
2778 i915 = rcu_dereference(ips_mchdev);
2779 if (i915 && !kref_get_unless_zero(kref: &i915->drm.ref))
2780 i915 = NULL;
2781 rcu_read_unlock();
2782
2783 return i915;
2784}
2785
2786/**
2787 * i915_read_mch_val - return value for IPS use
2788 *
2789 * Calculate and return a value for the IPS driver to use when deciding whether
2790 * we have thermal and power headroom to increase CPU or GPU power budget.
2791 */
2792unsigned long i915_read_mch_val(void)
2793{
2794 struct drm_i915_private *i915;
2795 unsigned long chipset_val = 0;
2796 unsigned long graphics_val = 0;
2797 intel_wakeref_t wakeref;
2798
2799 i915 = mchdev_get();
2800 if (!i915)
2801 return 0;
2802
2803 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2804 struct intel_ips *ips = &to_gt(i915)->rps.ips;
2805
2806 spin_lock_irq(lock: &mchdev_lock);
2807 chipset_val = __ips_chipset_val(ips);
2808 graphics_val = __ips_gfx_val(ips);
2809 spin_unlock_irq(lock: &mchdev_lock);
2810 }
2811
2812 drm_dev_put(dev: &i915->drm);
2813 return chipset_val + graphics_val;
2814}
2815EXPORT_SYMBOL_GPL(i915_read_mch_val);
2816
2817/**
2818 * i915_gpu_raise - raise GPU frequency limit
2819 *
2820 * Raise the limit; IPS indicates we have thermal headroom.
2821 */
2822bool i915_gpu_raise(void)
2823{
2824 struct drm_i915_private *i915;
2825 struct intel_rps *rps;
2826
2827 i915 = mchdev_get();
2828 if (!i915)
2829 return false;
2830
2831 rps = &to_gt(i915)->rps;
2832
2833 spin_lock_irq(lock: &mchdev_lock);
2834 if (rps->max_freq_softlimit < rps->max_freq)
2835 rps->max_freq_softlimit++;
2836 spin_unlock_irq(lock: &mchdev_lock);
2837
2838 drm_dev_put(dev: &i915->drm);
2839 return true;
2840}
2841EXPORT_SYMBOL_GPL(i915_gpu_raise);
2842
2843/**
2844 * i915_gpu_lower - lower GPU frequency limit
2845 *
2846 * IPS indicates we're close to a thermal limit, so throttle back the GPU
2847 * frequency maximum.
2848 */
2849bool i915_gpu_lower(void)
2850{
2851 struct drm_i915_private *i915;
2852 struct intel_rps *rps;
2853
2854 i915 = mchdev_get();
2855 if (!i915)
2856 return false;
2857
2858 rps = &to_gt(i915)->rps;
2859
2860 spin_lock_irq(lock: &mchdev_lock);
2861 if (rps->max_freq_softlimit > rps->min_freq)
2862 rps->max_freq_softlimit--;
2863 spin_unlock_irq(lock: &mchdev_lock);
2864
2865 drm_dev_put(dev: &i915->drm);
2866 return true;
2867}
2868EXPORT_SYMBOL_GPL(i915_gpu_lower);
2869
2870/**
2871 * i915_gpu_busy - indicate GPU business to IPS
2872 *
2873 * Tell the IPS driver whether or not the GPU is busy.
2874 */
2875bool i915_gpu_busy(void)
2876{
2877 struct drm_i915_private *i915;
2878 bool ret;
2879
2880 i915 = mchdev_get();
2881 if (!i915)
2882 return false;
2883
2884 ret = to_gt(i915)->awake;
2885
2886 drm_dev_put(dev: &i915->drm);
2887 return ret;
2888}
2889EXPORT_SYMBOL_GPL(i915_gpu_busy);
2890
2891/**
2892 * i915_gpu_turbo_disable - disable graphics turbo
2893 *
2894 * Disable graphics turbo by resetting the max frequency and setting the
2895 * current frequency to the default.
2896 */
2897bool i915_gpu_turbo_disable(void)
2898{
2899 struct drm_i915_private *i915;
2900 struct intel_rps *rps;
2901 bool ret;
2902
2903 i915 = mchdev_get();
2904 if (!i915)
2905 return false;
2906
2907 rps = &to_gt(i915)->rps;
2908
2909 spin_lock_irq(lock: &mchdev_lock);
2910 rps->max_freq_softlimit = rps->min_freq;
2911 ret = !__gen5_rps_set(rps: &to_gt(i915)->rps, val: rps->min_freq);
2912 spin_unlock_irq(lock: &mchdev_lock);
2913
2914 drm_dev_put(dev: &i915->drm);
2915 return ret;
2916}
2917EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2918
2919#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2920#include "selftest_rps.c"
2921#include "selftest_slpc.c"
2922#endif
2923