1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <drm/drm_atomic_state_helper.h>
7
8#include "soc/intel_dram.h"
9
10#include "i915_drv.h"
11#include "i915_reg.h"
12#include "i915_utils.h"
13#include "intel_atomic.h"
14#include "intel_bw.h"
15#include "intel_cdclk.h"
16#include "intel_display_core.h"
17#include "intel_display_regs.h"
18#include "intel_display_types.h"
19#include "intel_mchbar_regs.h"
20#include "intel_pcode.h"
21#include "intel_uncore.h"
22#include "skl_watermark.h"
23
24struct intel_dbuf_bw {
25 unsigned int max_bw[I915_MAX_DBUF_SLICES];
26 u8 active_planes[I915_MAX_DBUF_SLICES];
27};
28
29struct intel_bw_state {
30 struct intel_global_state base;
31 struct intel_dbuf_bw dbuf_bw[I915_MAX_PIPES];
32
33 /*
34 * Contains a bit mask, used to determine, whether correspondent
35 * pipe allows SAGV or not.
36 */
37 u8 pipe_sagv_reject;
38
39 /* bitmask of active pipes */
40 u8 active_pipes;
41
42 /*
43 * From MTL onwards, to lock a QGV point, punit expects the peak BW of
44 * the selected QGV point as the parameter in multiples of 100MB/s
45 */
46 u16 qgv_point_peakbw;
47
48 /*
49 * Current QGV points mask, which restricts
50 * some particular SAGV states, not to confuse
51 * with pipe_sagv_mask.
52 */
53 u16 qgv_points_mask;
54
55 unsigned int data_rate[I915_MAX_PIPES];
56 u8 num_active_planes[I915_MAX_PIPES];
57};
58
59/* Parameters for Qclk Geyserville (QGV) */
60struct intel_qgv_point {
61 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
62};
63
64#define DEPROGBWPCLIMIT 60
65
66struct intel_psf_gv_point {
67 u8 clk; /* clock in multiples of 16.6666 MHz */
68};
69
70struct intel_qgv_info {
71 struct intel_qgv_point points[I915_NUM_QGV_POINTS];
72 struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
73 u8 num_points;
74 u8 num_psf_points;
75 u8 t_bl;
76 u8 max_numchannels;
77 u8 channel_width;
78 u8 deinterleave;
79};
80
81static int dg1_mchbar_read_qgv_point_info(struct intel_display *display,
82 struct intel_qgv_point *sp,
83 int point)
84{
85 struct drm_i915_private *i915 = to_i915(dev: display->drm);
86 u32 dclk_ratio, dclk_reference;
87 u32 val;
88
89 val = intel_uncore_read(uncore: &i915->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
90 dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
91 if (val & DG1_QCLK_REFERENCE)
92 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
93 else
94 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
95 sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
96
97 val = intel_uncore_read(uncore: &i915->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
98 if (val & DG1_GEAR_TYPE)
99 sp->dclk *= 2;
100
101 if (sp->dclk == 0)
102 return -EINVAL;
103
104 val = intel_uncore_read(uncore: &i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
105 sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
106 sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
107
108 val = intel_uncore_read(uncore: &i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
109 sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
110 sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
111
112 sp->t_rc = sp->t_rp + sp->t_ras;
113
114 return 0;
115}
116
117static int icl_pcode_read_qgv_point_info(struct intel_display *display,
118 struct intel_qgv_point *sp,
119 int point)
120{
121 u32 val = 0, val2 = 0;
122 u16 dclk;
123 int ret;
124
125 ret = intel_pcode_read(drm: display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
126 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
127 val: &val, val1: &val2);
128 if (ret)
129 return ret;
130
131 dclk = val & 0xffff;
132 sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(display) >= 12 ? 500 : 0),
133 1000);
134 sp->t_rp = (val & 0xff0000) >> 16;
135 sp->t_rcd = (val & 0xff000000) >> 24;
136
137 sp->t_rdpre = val2 & 0xff;
138 sp->t_ras = (val2 & 0xff00) >> 8;
139
140 sp->t_rc = sp->t_rp + sp->t_ras;
141
142 return 0;
143}
144
145static int adls_pcode_read_psf_gv_point_info(struct intel_display *display,
146 struct intel_psf_gv_point *points)
147{
148 u32 val = 0;
149 int ret;
150 int i;
151
152 ret = intel_pcode_read(drm: display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
153 ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, val: &val, NULL);
154 if (ret)
155 return ret;
156
157 for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
158 points[i].clk = val & 0xff;
159 val >>= 8;
160 }
161
162 return 0;
163}
164
165static u16 icl_qgv_points_mask(struct intel_display *display)
166{
167 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
168 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
169 u16 qgv_points = 0, psf_points = 0;
170
171 /*
172 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
173 * it with failure if we try masking any unadvertised points.
174 * So need to operate only with those returned from PCode.
175 */
176 if (num_qgv_points > 0)
177 qgv_points = GENMASK(num_qgv_points - 1, 0);
178
179 if (num_psf_gv_points > 0)
180 psf_points = GENMASK(num_psf_gv_points - 1, 0);
181
182 return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
183}
184
185static bool is_sagv_enabled(struct intel_display *display, u16 points_mask)
186{
187 return !is_power_of_2(n: ~points_mask & icl_qgv_points_mask(display) &
188 ICL_PCODE_REQ_QGV_PT_MASK);
189}
190
191static int icl_pcode_restrict_qgv_points(struct intel_display *display,
192 u32 points_mask)
193{
194 int ret;
195
196 if (DISPLAY_VER(display) >= 14)
197 return 0;
198
199 /* bspec says to keep retrying for at least 1 ms */
200 ret = intel_pcode_request(drm: display->drm, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
201 request: points_mask,
202 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
203 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
204 timeout_base_ms: 1);
205
206 if (ret < 0) {
207 drm_err(display->drm,
208 "Failed to disable qgv points (0x%x) points: 0x%x\n",
209 ret, points_mask);
210 return ret;
211 }
212
213 display->sagv.status = is_sagv_enabled(display, points_mask) ?
214 I915_SAGV_ENABLED : I915_SAGV_DISABLED;
215
216 return 0;
217}
218
219static int mtl_read_qgv_point_info(struct intel_display *display,
220 struct intel_qgv_point *sp, int point)
221{
222 struct drm_i915_private *i915 = to_i915(dev: display->drm);
223 u32 val, val2;
224 u16 dclk;
225
226 val = intel_uncore_read(uncore: &i915->uncore,
227 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
228 val2 = intel_uncore_read(uncore: &i915->uncore,
229 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
230 dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
231 sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
232 sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
233 sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
234
235 sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
236 sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
237
238 sp->t_rc = sp->t_rp + sp->t_ras;
239
240 return 0;
241}
242
243static int
244intel_read_qgv_point_info(struct intel_display *display,
245 struct intel_qgv_point *sp,
246 int point)
247{
248 if (DISPLAY_VER(display) >= 14)
249 return mtl_read_qgv_point_info(display, sp, point);
250 else if (display->platform.dg1)
251 return dg1_mchbar_read_qgv_point_info(display, sp, point);
252 else
253 return icl_pcode_read_qgv_point_info(display, sp, point);
254}
255
256static int icl_get_qgv_points(struct intel_display *display,
257 const struct dram_info *dram_info,
258 struct intel_qgv_info *qi,
259 bool is_y_tile)
260{
261 int i, ret;
262
263 qi->num_points = dram_info->num_qgv_points;
264 qi->num_psf_points = dram_info->num_psf_gv_points;
265
266 if (DISPLAY_VER(display) >= 14) {
267 switch (dram_info->type) {
268 case INTEL_DRAM_DDR4:
269 qi->t_bl = 4;
270 qi->max_numchannels = 2;
271 qi->channel_width = 64;
272 qi->deinterleave = 2;
273 break;
274 case INTEL_DRAM_DDR5:
275 qi->t_bl = 8;
276 qi->max_numchannels = 4;
277 qi->channel_width = 32;
278 qi->deinterleave = 2;
279 break;
280 case INTEL_DRAM_LPDDR4:
281 case INTEL_DRAM_LPDDR5:
282 qi->t_bl = 16;
283 qi->max_numchannels = 8;
284 qi->channel_width = 16;
285 qi->deinterleave = 4;
286 break;
287 case INTEL_DRAM_GDDR:
288 case INTEL_DRAM_GDDR_ECC:
289 qi->channel_width = 32;
290 break;
291 default:
292 MISSING_CASE(dram_info->type);
293 return -EINVAL;
294 }
295 } else if (DISPLAY_VER(display) >= 12) {
296 switch (dram_info->type) {
297 case INTEL_DRAM_DDR4:
298 qi->t_bl = is_y_tile ? 8 : 4;
299 qi->max_numchannels = 2;
300 qi->channel_width = 64;
301 qi->deinterleave = is_y_tile ? 1 : 2;
302 break;
303 case INTEL_DRAM_DDR5:
304 qi->t_bl = is_y_tile ? 16 : 8;
305 qi->max_numchannels = 4;
306 qi->channel_width = 32;
307 qi->deinterleave = is_y_tile ? 1 : 2;
308 break;
309 case INTEL_DRAM_LPDDR4:
310 if (display->platform.rocketlake) {
311 qi->t_bl = 8;
312 qi->max_numchannels = 4;
313 qi->channel_width = 32;
314 qi->deinterleave = 2;
315 break;
316 }
317 fallthrough;
318 case INTEL_DRAM_LPDDR5:
319 qi->t_bl = 16;
320 qi->max_numchannels = 8;
321 qi->channel_width = 16;
322 qi->deinterleave = is_y_tile ? 2 : 4;
323 break;
324 default:
325 qi->t_bl = 16;
326 qi->max_numchannels = 1;
327 break;
328 }
329 } else if (DISPLAY_VER(display) == 11) {
330 qi->t_bl = dram_info->type == INTEL_DRAM_DDR4 ? 4 : 8;
331 qi->max_numchannels = 1;
332 }
333
334 if (drm_WARN_ON(display->drm,
335 qi->num_points > ARRAY_SIZE(qi->points)))
336 qi->num_points = ARRAY_SIZE(qi->points);
337
338 for (i = 0; i < qi->num_points; i++) {
339 struct intel_qgv_point *sp = &qi->points[i];
340
341 ret = intel_read_qgv_point_info(display, sp, point: i);
342 if (ret) {
343 drm_dbg_kms(display->drm, "Could not read QGV %d info\n", i);
344 return ret;
345 }
346
347 drm_dbg_kms(display->drm,
348 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
349 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
350 sp->t_rcd, sp->t_rc);
351 }
352
353 if (qi->num_psf_points > 0) {
354 ret = adls_pcode_read_psf_gv_point_info(display, points: qi->psf_points);
355 if (ret) {
356 drm_err(display->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
357 qi->num_psf_points = 0;
358 }
359
360 for (i = 0; i < qi->num_psf_points; i++)
361 drm_dbg_kms(display->drm,
362 "PSF GV %d: CLK=%d\n",
363 i, qi->psf_points[i].clk);
364 }
365
366 return 0;
367}
368
369static int adl_calc_psf_bw(int clk)
370{
371 /*
372 * clk is multiples of 16.666MHz (100/6)
373 * According to BSpec PSF GV bandwidth is
374 * calculated as BW = 64 * clk * 16.666Mhz
375 */
376 return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
377}
378
379static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
380{
381 u16 dclk = 0;
382 int i;
383
384 for (i = 0; i < qi->num_points; i++)
385 dclk = max(dclk, qi->points[i].dclk);
386
387 return dclk;
388}
389
390struct intel_sa_info {
391 u16 displayrtids;
392 u8 deburst, deprogbwlimit, derating;
393};
394
395static const struct intel_sa_info icl_sa_info = {
396 .deburst = 8,
397 .deprogbwlimit = 25, /* GB/s */
398 .displayrtids = 128,
399 .derating = 10,
400};
401
402static const struct intel_sa_info tgl_sa_info = {
403 .deburst = 16,
404 .deprogbwlimit = 34, /* GB/s */
405 .displayrtids = 256,
406 .derating = 10,
407};
408
409static const struct intel_sa_info rkl_sa_info = {
410 .deburst = 8,
411 .deprogbwlimit = 20, /* GB/s */
412 .displayrtids = 128,
413 .derating = 10,
414};
415
416static const struct intel_sa_info adls_sa_info = {
417 .deburst = 16,
418 .deprogbwlimit = 38, /* GB/s */
419 .displayrtids = 256,
420 .derating = 10,
421};
422
423static const struct intel_sa_info adlp_sa_info = {
424 .deburst = 16,
425 .deprogbwlimit = 38, /* GB/s */
426 .displayrtids = 256,
427 .derating = 20,
428};
429
430static const struct intel_sa_info mtl_sa_info = {
431 .deburst = 32,
432 .deprogbwlimit = 38, /* GB/s */
433 .displayrtids = 256,
434 .derating = 10,
435};
436
437static const struct intel_sa_info xe2_hpd_sa_info = {
438 .derating = 30,
439 .deprogbwlimit = 53,
440 /* Other values not used by simplified algorithm */
441};
442
443static const struct intel_sa_info xe2_hpd_ecc_sa_info = {
444 .derating = 45,
445 .deprogbwlimit = 53,
446 /* Other values not used by simplified algorithm */
447};
448
449static const struct intel_sa_info xe3lpd_sa_info = {
450 .deburst = 32,
451 .deprogbwlimit = 65, /* GB/s */
452 .displayrtids = 256,
453 .derating = 10,
454};
455
456static const struct intel_sa_info xe3lpd_3002_sa_info = {
457 .deburst = 32,
458 .deprogbwlimit = 22, /* GB/s */
459 .displayrtids = 256,
460 .derating = 10,
461};
462
463static int icl_get_bw_info(struct intel_display *display,
464 const struct dram_info *dram_info,
465 const struct intel_sa_info *sa)
466{
467 struct intel_qgv_info qi = {};
468 bool is_y_tile = true; /* assume y tile may be used */
469 int num_channels = max_t(u8, 1, dram_info->num_channels);
470 int ipqdepth, ipqdepthpch = 16;
471 int dclk_max;
472 int maxdebw;
473 int num_groups = ARRAY_SIZE(display->bw.max);
474 int i, ret;
475
476 ret = icl_get_qgv_points(display, dram_info, qi: &qi, is_y_tile);
477 if (ret) {
478 drm_dbg_kms(display->drm,
479 "Failed to get memory subsystem information, ignoring bandwidth limits");
480 return ret;
481 }
482
483 dclk_max = icl_sagv_max_dclk(qi: &qi);
484 maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
485 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
486 qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
487
488 for (i = 0; i < num_groups; i++) {
489 struct intel_bw_info *bi = &display->bw.max[i];
490 int clpchgroup;
491 int j;
492
493 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
494 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
495
496 bi->num_qgv_points = qi.num_points;
497 bi->num_psf_gv_points = qi.num_psf_points;
498
499 for (j = 0; j < qi.num_points; j++) {
500 const struct intel_qgv_point *sp = &qi.points[j];
501 int ct, bw;
502
503 /*
504 * Max row cycle time
505 *
506 * FIXME what is the logic behind the
507 * assumed burst length?
508 */
509 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
510 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
511 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
512
513 bi->deratedbw[j] = min(maxdebw,
514 bw * (100 - sa->derating) / 100);
515
516 drm_dbg_kms(display->drm,
517 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
518 i, j, bi->num_planes, bi->deratedbw[j]);
519 }
520 }
521 /*
522 * In case if SAGV is disabled in BIOS, we always get 1
523 * SAGV point, but we can't send PCode commands to restrict it
524 * as it will fail and pointless anyway.
525 */
526 if (qi.num_points == 1)
527 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
528 else
529 display->sagv.status = I915_SAGV_ENABLED;
530
531 return 0;
532}
533
534static int tgl_get_bw_info(struct intel_display *display,
535 const struct dram_info *dram_info,
536 const struct intel_sa_info *sa)
537{
538 struct intel_qgv_info qi = {};
539 bool is_y_tile = true; /* assume y tile may be used */
540 int num_channels = max_t(u8, 1, dram_info->num_channels);
541 int ipqdepth, ipqdepthpch = 16;
542 int dclk_max;
543 int maxdebw, peakbw;
544 int clperchgroup;
545 int num_groups = ARRAY_SIZE(display->bw.max);
546 int i, ret;
547
548 ret = icl_get_qgv_points(display, dram_info, qi: &qi, is_y_tile);
549 if (ret) {
550 drm_dbg_kms(display->drm,
551 "Failed to get memory subsystem information, ignoring bandwidth limits");
552 return ret;
553 }
554
555 if (DISPLAY_VER(display) < 14 &&
556 (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
557 num_channels *= 2;
558
559 qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
560
561 if (num_channels < qi.max_numchannels && DISPLAY_VER(display) >= 12)
562 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
563
564 if (DISPLAY_VER(display) >= 12 && num_channels > qi.max_numchannels)
565 drm_warn(display->drm, "Number of channels exceeds max number of channels.");
566 if (qi.max_numchannels != 0)
567 num_channels = min_t(u8, num_channels, qi.max_numchannels);
568
569 dclk_max = icl_sagv_max_dclk(qi: &qi);
570
571 peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
572 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);
573
574 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
575 /*
576 * clperchgroup = 4kpagespermempage * clperchperblock,
577 * clperchperblock = 8 / num_channels * interleave
578 */
579 clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
580
581 for (i = 0; i < num_groups; i++) {
582 struct intel_bw_info *bi = &display->bw.max[i];
583 struct intel_bw_info *bi_next;
584 int clpchgroup;
585 int j;
586
587 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
588
589 if (i < num_groups - 1) {
590 bi_next = &display->bw.max[i + 1];
591
592 if (clpchgroup < clperchgroup)
593 bi_next->num_planes = (ipqdepth - clpchgroup) /
594 clpchgroup + 1;
595 else
596 bi_next->num_planes = 0;
597 }
598
599 bi->num_qgv_points = qi.num_points;
600 bi->num_psf_gv_points = qi.num_psf_points;
601
602 for (j = 0; j < qi.num_points; j++) {
603 const struct intel_qgv_point *sp = &qi.points[j];
604 int ct, bw;
605
606 /*
607 * Max row cycle time
608 *
609 * FIXME what is the logic behind the
610 * assumed burst length?
611 */
612 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
613 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
614 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
615
616 bi->deratedbw[j] = min(maxdebw,
617 bw * (100 - sa->derating) / 100);
618 bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
619 num_channels *
620 qi.channel_width, 8);
621
622 drm_dbg_kms(display->drm,
623 "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
624 i, j, bi->num_planes, bi->deratedbw[j],
625 bi->peakbw[j]);
626 }
627
628 for (j = 0; j < qi.num_psf_points; j++) {
629 const struct intel_psf_gv_point *sp = &qi.psf_points[j];
630
631 bi->psf_bw[j] = adl_calc_psf_bw(clk: sp->clk);
632
633 drm_dbg_kms(display->drm,
634 "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
635 i, j, bi->num_planes, bi->psf_bw[j]);
636 }
637 }
638
639 /*
640 * In case if SAGV is disabled in BIOS, we always get 1
641 * SAGV point, but we can't send PCode commands to restrict it
642 * as it will fail and pointless anyway.
643 */
644 if (qi.num_points == 1)
645 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
646 else
647 display->sagv.status = I915_SAGV_ENABLED;
648
649 return 0;
650}
651
652static void dg2_get_bw_info(struct intel_display *display)
653{
654 unsigned int deratedbw = display->platform.dg2_g11 ? 38000 : 50000;
655 int num_groups = ARRAY_SIZE(display->bw.max);
656 int i;
657
658 /*
659 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
660 * that doesn't depend on the number of planes enabled. So fill all the
661 * plane group with constant bw information for uniformity with other
662 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
663 * whereas DG2-G11 platforms have 38 GB/s.
664 */
665 for (i = 0; i < num_groups; i++) {
666 struct intel_bw_info *bi = &display->bw.max[i];
667
668 bi->num_planes = 1;
669 /* Need only one dummy QGV point per group */
670 bi->num_qgv_points = 1;
671 bi->deratedbw[0] = deratedbw;
672 }
673
674 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
675}
676
677static int xe2_hpd_get_bw_info(struct intel_display *display,
678 const struct dram_info *dram_info,
679 const struct intel_sa_info *sa)
680{
681 struct intel_qgv_info qi = {};
682 int num_channels = dram_info->num_channels;
683 int peakbw, maxdebw;
684 int ret, i;
685
686 ret = icl_get_qgv_points(display, dram_info, qi: &qi, is_y_tile: true);
687 if (ret) {
688 drm_dbg_kms(display->drm,
689 "Failed to get memory subsystem information, ignoring bandwidth limits");
690 return ret;
691 }
692
693 peakbw = num_channels * qi.channel_width / 8 * icl_sagv_max_dclk(qi: &qi);
694 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 10);
695
696 for (i = 0; i < qi.num_points; i++) {
697 const struct intel_qgv_point *point = &qi.points[i];
698 int bw = num_channels * (qi.channel_width / 8) * point->dclk;
699
700 display->bw.max[0].deratedbw[i] =
701 min(maxdebw, (100 - sa->derating) * bw / 100);
702 display->bw.max[0].peakbw[i] = bw;
703
704 drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw: %u\n",
705 i, display->bw.max[0].deratedbw[i],
706 display->bw.max[0].peakbw[i]);
707 }
708
709 /* Bandwidth does not depend on # of planes; set all groups the same */
710 display->bw.max[0].num_planes = 1;
711 display->bw.max[0].num_qgv_points = qi.num_points;
712 for (i = 1; i < ARRAY_SIZE(display->bw.max); i++)
713 memcpy(to: &display->bw.max[i], from: &display->bw.max[0],
714 len: sizeof(display->bw.max[0]));
715
716 /*
717 * Xe2_HPD should always have exactly two QGV points representing
718 * battery and plugged-in operation.
719 */
720 drm_WARN_ON(display->drm, qi.num_points != 2);
721 display->sagv.status = I915_SAGV_ENABLED;
722
723 return 0;
724}
725
726static unsigned int icl_max_bw_index(struct intel_display *display,
727 int num_planes, int qgv_point)
728{
729 int i;
730
731 /*
732 * Let's return max bw for 0 planes
733 */
734 num_planes = max(1, num_planes);
735
736 for (i = 0; i < ARRAY_SIZE(display->bw.max); i++) {
737 const struct intel_bw_info *bi =
738 &display->bw.max[i];
739
740 /*
741 * Pcode will not expose all QGV points when
742 * SAGV is forced to off/min/med/max.
743 */
744 if (qgv_point >= bi->num_qgv_points)
745 return UINT_MAX;
746
747 if (num_planes >= bi->num_planes)
748 return i;
749 }
750
751 return UINT_MAX;
752}
753
754static unsigned int tgl_max_bw_index(struct intel_display *display,
755 int num_planes, int qgv_point)
756{
757 int i;
758
759 /*
760 * Let's return max bw for 0 planes
761 */
762 num_planes = max(1, num_planes);
763
764 for (i = ARRAY_SIZE(display->bw.max) - 1; i >= 0; i--) {
765 const struct intel_bw_info *bi =
766 &display->bw.max[i];
767
768 /*
769 * Pcode will not expose all QGV points when
770 * SAGV is forced to off/min/med/max.
771 */
772 if (qgv_point >= bi->num_qgv_points)
773 return UINT_MAX;
774
775 if (num_planes <= bi->num_planes)
776 return i;
777 }
778
779 return 0;
780}
781
782static unsigned int adl_psf_bw(struct intel_display *display,
783 int psf_gv_point)
784{
785 const struct intel_bw_info *bi =
786 &display->bw.max[0];
787
788 return bi->psf_bw[psf_gv_point];
789}
790
791static unsigned int icl_qgv_bw(struct intel_display *display,
792 int num_active_planes, int qgv_point)
793{
794 unsigned int idx;
795
796 if (DISPLAY_VER(display) >= 12)
797 idx = tgl_max_bw_index(display, num_planes: num_active_planes, qgv_point);
798 else
799 idx = icl_max_bw_index(display, num_planes: num_active_planes, qgv_point);
800
801 if (idx >= ARRAY_SIZE(display->bw.max))
802 return 0;
803
804 return display->bw.max[idx].deratedbw[qgv_point];
805}
806
807void intel_bw_init_hw(struct intel_display *display)
808{
809 const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
810
811 if (!HAS_DISPLAY(display))
812 return;
813
814 if (DISPLAY_VERx100(display) >= 3002)
815 tgl_get_bw_info(display, dram_info, sa: &xe3lpd_3002_sa_info);
816 else if (DISPLAY_VER(display) >= 30)
817 tgl_get_bw_info(display, dram_info, sa: &xe3lpd_sa_info);
818 else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx &&
819 dram_info->type == INTEL_DRAM_GDDR_ECC)
820 xe2_hpd_get_bw_info(display, dram_info, sa: &xe2_hpd_ecc_sa_info);
821 else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx)
822 xe2_hpd_get_bw_info(display, dram_info, sa: &xe2_hpd_sa_info);
823 else if (DISPLAY_VER(display) >= 14)
824 tgl_get_bw_info(display, dram_info, sa: &mtl_sa_info);
825 else if (display->platform.dg2)
826 dg2_get_bw_info(display);
827 else if (display->platform.alderlake_p)
828 tgl_get_bw_info(display, dram_info, sa: &adlp_sa_info);
829 else if (display->platform.alderlake_s)
830 tgl_get_bw_info(display, dram_info, sa: &adls_sa_info);
831 else if (display->platform.rocketlake)
832 tgl_get_bw_info(display, dram_info, sa: &rkl_sa_info);
833 else if (DISPLAY_VER(display) == 12)
834 tgl_get_bw_info(display, dram_info, sa: &tgl_sa_info);
835 else if (DISPLAY_VER(display) == 11)
836 icl_get_bw_info(display, dram_info, sa: &icl_sa_info);
837}
838
839static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
840{
841 /*
842 * We assume cursors are small enough
843 * to not not cause bandwidth problems.
844 */
845 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
846}
847
848static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
849{
850 struct intel_display *display = to_intel_display(crtc_state);
851 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
852 unsigned int data_rate = 0;
853 enum plane_id plane_id;
854
855 for_each_plane_id_on_crtc(crtc, plane_id) {
856 /*
857 * We assume cursors are small enough
858 * to not not cause bandwidth problems.
859 */
860 if (plane_id == PLANE_CURSOR)
861 continue;
862
863 data_rate += crtc_state->data_rate[plane_id];
864
865 if (DISPLAY_VER(display) < 11)
866 data_rate += crtc_state->data_rate_y[plane_id];
867 }
868
869 return data_rate;
870}
871
872/* "Maximum Pipe Read Bandwidth" */
873static int intel_bw_crtc_min_cdclk(struct intel_display *display,
874 unsigned int data_rate)
875{
876 if (DISPLAY_VER(display) < 12)
877 return 0;
878
879 return DIV_ROUND_UP_ULL(mul_u32_u32(data_rate, 10), 512);
880}
881
882static unsigned int intel_bw_num_active_planes(struct intel_display *display,
883 const struct intel_bw_state *bw_state)
884{
885 unsigned int num_active_planes = 0;
886 enum pipe pipe;
887
888 for_each_pipe(display, pipe)
889 num_active_planes += bw_state->num_active_planes[pipe];
890
891 return num_active_planes;
892}
893
894static unsigned int intel_bw_data_rate(struct intel_display *display,
895 const struct intel_bw_state *bw_state)
896{
897 struct drm_i915_private *i915 = to_i915(dev: display->drm);
898 unsigned int data_rate = 0;
899 enum pipe pipe;
900
901 for_each_pipe(display, pipe)
902 data_rate += bw_state->data_rate[pipe];
903
904 if (DISPLAY_VER(display) >= 13 && i915_vtd_active(i915))
905 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
906
907 return data_rate;
908}
909
910struct intel_bw_state *to_intel_bw_state(struct intel_global_state *obj_state)
911{
912 return container_of(obj_state, struct intel_bw_state, base);
913}
914
915struct intel_bw_state *
916intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
917{
918 struct intel_display *display = to_intel_display(state);
919 struct intel_global_state *bw_state;
920
921 bw_state = intel_atomic_get_old_global_obj_state(state, obj: &display->bw.obj);
922
923 return to_intel_bw_state(obj_state: bw_state);
924}
925
926struct intel_bw_state *
927intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
928{
929 struct intel_display *display = to_intel_display(state);
930 struct intel_global_state *bw_state;
931
932 bw_state = intel_atomic_get_new_global_obj_state(state, obj: &display->bw.obj);
933
934 return to_intel_bw_state(obj_state: bw_state);
935}
936
937struct intel_bw_state *
938intel_atomic_get_bw_state(struct intel_atomic_state *state)
939{
940 struct intel_display *display = to_intel_display(state);
941 struct intel_global_state *bw_state;
942
943 bw_state = intel_atomic_get_global_obj_state(state, obj: &display->bw.obj);
944 if (IS_ERR(ptr: bw_state))
945 return ERR_CAST(ptr: bw_state);
946
947 return to_intel_bw_state(obj_state: bw_state);
948}
949
950static unsigned int icl_max_bw_qgv_point_mask(struct intel_display *display,
951 int num_active_planes)
952{
953 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
954 unsigned int max_bw_point = 0;
955 unsigned int max_bw = 0;
956 int i;
957
958 for (i = 0; i < num_qgv_points; i++) {
959 unsigned int max_data_rate =
960 icl_qgv_bw(display, num_active_planes, qgv_point: i);
961
962 /*
963 * We need to know which qgv point gives us
964 * maximum bandwidth in order to disable SAGV
965 * if we find that we exceed SAGV block time
966 * with watermarks. By that moment we already
967 * have those, as it is calculated earlier in
968 * intel_atomic_check,
969 */
970 if (max_data_rate > max_bw) {
971 max_bw_point = BIT(i);
972 max_bw = max_data_rate;
973 }
974 }
975
976 return max_bw_point;
977}
978
979static u16 icl_prepare_qgv_points_mask(struct intel_display *display,
980 unsigned int qgv_points,
981 unsigned int psf_points)
982{
983 return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
984 ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(display);
985}
986
987static unsigned int icl_max_bw_psf_gv_point_mask(struct intel_display *display)
988{
989 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
990 unsigned int max_bw_point_mask = 0;
991 unsigned int max_bw = 0;
992 int i;
993
994 for (i = 0; i < num_psf_gv_points; i++) {
995 unsigned int max_data_rate = adl_psf_bw(display, psf_gv_point: i);
996
997 if (max_data_rate > max_bw) {
998 max_bw_point_mask = BIT(i);
999 max_bw = max_data_rate;
1000 } else if (max_data_rate == max_bw) {
1001 max_bw_point_mask |= BIT(i);
1002 }
1003 }
1004
1005 return max_bw_point_mask;
1006}
1007
1008static void icl_force_disable_sagv(struct intel_display *display,
1009 struct intel_bw_state *bw_state)
1010{
1011 unsigned int qgv_points = icl_max_bw_qgv_point_mask(display, num_active_planes: 0);
1012 unsigned int psf_points = icl_max_bw_psf_gv_point_mask(display);
1013
1014 bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
1015 qgv_points,
1016 psf_points);
1017
1018 drm_dbg_kms(display->drm, "Forcing SAGV disable: mask 0x%x\n",
1019 bw_state->qgv_points_mask);
1020
1021 icl_pcode_restrict_qgv_points(display, points_mask: bw_state->qgv_points_mask);
1022}
1023
1024void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
1025{
1026 struct intel_display *display = to_intel_display(state);
1027 const struct intel_bw_state *old_bw_state =
1028 intel_atomic_get_old_bw_state(state);
1029 const struct intel_bw_state *new_bw_state =
1030 intel_atomic_get_new_bw_state(state);
1031 u16 old_mask, new_mask;
1032
1033 if (!new_bw_state)
1034 return;
1035
1036 old_mask = old_bw_state->qgv_points_mask;
1037 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
1038
1039 if (old_mask == new_mask)
1040 return;
1041
1042 WARN_ON(!new_bw_state->base.changed);
1043
1044 drm_dbg_kms(display->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
1045 old_mask, new_mask);
1046
1047 /*
1048 * Restrict required qgv points before updating the configuration.
1049 * According to BSpec we can't mask and unmask qgv points at the same
1050 * time. Also masking should be done before updating the configuration
1051 * and unmasking afterwards.
1052 */
1053 icl_pcode_restrict_qgv_points(display, points_mask: new_mask);
1054}
1055
1056void icl_sagv_post_plane_update(struct intel_atomic_state *state)
1057{
1058 struct intel_display *display = to_intel_display(state);
1059 const struct intel_bw_state *old_bw_state =
1060 intel_atomic_get_old_bw_state(state);
1061 const struct intel_bw_state *new_bw_state =
1062 intel_atomic_get_new_bw_state(state);
1063 u16 old_mask, new_mask;
1064
1065 if (!new_bw_state)
1066 return;
1067
1068 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
1069 new_mask = new_bw_state->qgv_points_mask;
1070
1071 if (old_mask == new_mask)
1072 return;
1073
1074 WARN_ON(!new_bw_state->base.changed);
1075
1076 drm_dbg_kms(display->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
1077 old_mask, new_mask);
1078
1079 /*
1080 * Allow required qgv points after updating the configuration.
1081 * According to BSpec we can't mask and unmask qgv points at the same
1082 * time. Also masking should be done before updating the configuration
1083 * and unmasking afterwards.
1084 */
1085 icl_pcode_restrict_qgv_points(display, points_mask: new_mask);
1086}
1087
1088static int mtl_find_qgv_points(struct intel_display *display,
1089 unsigned int data_rate,
1090 unsigned int num_active_planes,
1091 struct intel_bw_state *new_bw_state)
1092{
1093 unsigned int best_rate = UINT_MAX;
1094 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
1095 unsigned int qgv_peak_bw = 0;
1096 int i;
1097 int ret;
1098
1099 ret = intel_atomic_lock_global_state(obj_state: &new_bw_state->base);
1100 if (ret)
1101 return ret;
1102
1103 /*
1104 * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
1105 * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
1106 * not enabled. PM Demand code will clamp the value for the register
1107 */
1108 if (!intel_bw_can_enable_sagv(display, bw_state: new_bw_state)) {
1109 new_bw_state->qgv_point_peakbw = U16_MAX;
1110 drm_dbg_kms(display->drm, "No SAGV, use UINT_MAX as peak bw.");
1111 return 0;
1112 }
1113
1114 /*
1115 * Find the best QGV point by comparing the data_rate with max data rate
1116 * offered per plane group
1117 */
1118 for (i = 0; i < num_qgv_points; i++) {
1119 unsigned int bw_index =
1120 tgl_max_bw_index(display, num_planes: num_active_planes, qgv_point: i);
1121 unsigned int max_data_rate;
1122
1123 if (bw_index >= ARRAY_SIZE(display->bw.max))
1124 continue;
1125
1126 max_data_rate = display->bw.max[bw_index].deratedbw[i];
1127
1128 if (max_data_rate < data_rate)
1129 continue;
1130
1131 if (max_data_rate - data_rate < best_rate) {
1132 best_rate = max_data_rate - data_rate;
1133 qgv_peak_bw = display->bw.max[bw_index].peakbw[i];
1134 }
1135
1136 drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
1137 i, max_data_rate, data_rate, qgv_peak_bw);
1138 }
1139
1140 drm_dbg_kms(display->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
1141 qgv_peak_bw, data_rate);
1142
1143 /*
1144 * The display configuration cannot be supported if no QGV point
1145 * satisfying the required data rate is found
1146 */
1147 if (qgv_peak_bw == 0) {
1148 drm_dbg_kms(display->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
1149 data_rate, num_active_planes);
1150 return -EINVAL;
1151 }
1152
1153 /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
1154 new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
1155
1156 return 0;
1157}
1158
1159static int icl_find_qgv_points(struct intel_display *display,
1160 unsigned int data_rate,
1161 unsigned int num_active_planes,
1162 const struct intel_bw_state *old_bw_state,
1163 struct intel_bw_state *new_bw_state)
1164{
1165 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
1166 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
1167 u16 psf_points = 0;
1168 u16 qgv_points = 0;
1169 int i;
1170 int ret;
1171
1172 ret = intel_atomic_lock_global_state(obj_state: &new_bw_state->base);
1173 if (ret)
1174 return ret;
1175
1176 for (i = 0; i < num_qgv_points; i++) {
1177 unsigned int max_data_rate = icl_qgv_bw(display,
1178 num_active_planes, qgv_point: i);
1179 if (max_data_rate >= data_rate)
1180 qgv_points |= BIT(i);
1181
1182 drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d\n",
1183 i, max_data_rate, data_rate);
1184 }
1185
1186 for (i = 0; i < num_psf_gv_points; i++) {
1187 unsigned int max_data_rate = adl_psf_bw(display, psf_gv_point: i);
1188
1189 if (max_data_rate >= data_rate)
1190 psf_points |= BIT(i);
1191
1192 drm_dbg_kms(display->drm, "PSF GV point %d: max bw %d"
1193 " required %d\n",
1194 i, max_data_rate, data_rate);
1195 }
1196
1197 /*
1198 * BSpec states that we always should have at least one allowed point
1199 * left, so if we couldn't - simply reject the configuration for obvious
1200 * reasons.
1201 */
1202 if (qgv_points == 0) {
1203 drm_dbg_kms(display->drm, "No QGV points provide sufficient memory"
1204 " bandwidth %d for display configuration(%d active planes).\n",
1205 data_rate, num_active_planes);
1206 return -EINVAL;
1207 }
1208
1209 if (num_psf_gv_points > 0 && psf_points == 0) {
1210 drm_dbg_kms(display->drm, "No PSF GV points provide sufficient memory"
1211 " bandwidth %d for display configuration(%d active planes).\n",
1212 data_rate, num_active_planes);
1213 return -EINVAL;
1214 }
1215
1216 /*
1217 * Leave only single point with highest bandwidth, if
1218 * we can't enable SAGV due to the increased memory latency it may
1219 * cause.
1220 */
1221 if (!intel_bw_can_enable_sagv(display, bw_state: new_bw_state)) {
1222 qgv_points = icl_max_bw_qgv_point_mask(display, num_active_planes);
1223 drm_dbg_kms(display->drm, "No SAGV, using single QGV point mask 0x%x\n",
1224 qgv_points);
1225 }
1226
1227 /*
1228 * We store the ones which need to be masked as that is what PCode
1229 * actually accepts as a parameter.
1230 */
1231 new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
1232 qgv_points,
1233 psf_points);
1234 /*
1235 * If the actual mask had changed we need to make sure that
1236 * the commits are serialized(in case this is a nomodeset, nonblocking)
1237 */
1238 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1239 ret = intel_atomic_serialize_global_state(obj_state: &new_bw_state->base);
1240 if (ret)
1241 return ret;
1242 }
1243
1244 return 0;
1245}
1246
1247static int intel_bw_check_qgv_points(struct intel_display *display,
1248 const struct intel_bw_state *old_bw_state,
1249 struct intel_bw_state *new_bw_state)
1250{
1251 unsigned int data_rate = intel_bw_data_rate(display, bw_state: new_bw_state);
1252 unsigned int num_active_planes =
1253 intel_bw_num_active_planes(display, bw_state: new_bw_state);
1254
1255 data_rate = DIV_ROUND_UP(data_rate, 1000);
1256
1257 if (DISPLAY_VER(display) >= 14)
1258 return mtl_find_qgv_points(display, data_rate, num_active_planes,
1259 new_bw_state);
1260 else
1261 return icl_find_qgv_points(display, data_rate, num_active_planes,
1262 old_bw_state, new_bw_state);
1263}
1264
1265static bool intel_dbuf_bw_changed(struct intel_display *display,
1266 const struct intel_dbuf_bw *old_dbuf_bw,
1267 const struct intel_dbuf_bw *new_dbuf_bw)
1268{
1269 enum dbuf_slice slice;
1270
1271 for_each_dbuf_slice(display, slice) {
1272 if (old_dbuf_bw->max_bw[slice] != new_dbuf_bw->max_bw[slice] ||
1273 old_dbuf_bw->active_planes[slice] != new_dbuf_bw->active_planes[slice])
1274 return true;
1275 }
1276
1277 return false;
1278}
1279
1280static bool intel_bw_state_changed(struct intel_display *display,
1281 const struct intel_bw_state *old_bw_state,
1282 const struct intel_bw_state *new_bw_state)
1283{
1284 enum pipe pipe;
1285
1286 for_each_pipe(display, pipe) {
1287 const struct intel_dbuf_bw *old_dbuf_bw =
1288 &old_bw_state->dbuf_bw[pipe];
1289 const struct intel_dbuf_bw *new_dbuf_bw =
1290 &new_bw_state->dbuf_bw[pipe];
1291
1292 if (intel_dbuf_bw_changed(display, old_dbuf_bw, new_dbuf_bw))
1293 return true;
1294
1295 if (intel_bw_crtc_min_cdclk(display, data_rate: old_bw_state->data_rate[pipe]) !=
1296 intel_bw_crtc_min_cdclk(display, data_rate: new_bw_state->data_rate[pipe]))
1297 return true;
1298 }
1299
1300 return false;
1301}
1302
1303static void skl_plane_calc_dbuf_bw(struct intel_dbuf_bw *dbuf_bw,
1304 struct intel_crtc *crtc,
1305 enum plane_id plane_id,
1306 const struct skl_ddb_entry *ddb,
1307 unsigned int data_rate)
1308{
1309 struct intel_display *display = to_intel_display(crtc);
1310 unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(display, entry: ddb);
1311 enum dbuf_slice slice;
1312
1313 /*
1314 * The arbiter can only really guarantee an
1315 * equal share of the total bw to each plane.
1316 */
1317 for_each_dbuf_slice_in_mask(display, slice, dbuf_mask) {
1318 dbuf_bw->max_bw[slice] = max(dbuf_bw->max_bw[slice], data_rate);
1319 dbuf_bw->active_planes[slice] |= BIT(plane_id);
1320 }
1321}
1322
1323static void skl_crtc_calc_dbuf_bw(struct intel_dbuf_bw *dbuf_bw,
1324 const struct intel_crtc_state *crtc_state)
1325{
1326 struct intel_display *display = to_intel_display(crtc_state);
1327 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1328 enum plane_id plane_id;
1329
1330 memset(s: dbuf_bw, c: 0, n: sizeof(*dbuf_bw));
1331
1332 if (!crtc_state->hw.active)
1333 return;
1334
1335 for_each_plane_id_on_crtc(crtc, plane_id) {
1336 /*
1337 * We assume cursors are small enough
1338 * to not cause bandwidth problems.
1339 */
1340 if (plane_id == PLANE_CURSOR)
1341 continue;
1342
1343 skl_plane_calc_dbuf_bw(dbuf_bw, crtc, plane_id,
1344 ddb: &crtc_state->wm.skl.plane_ddb[plane_id],
1345 data_rate: crtc_state->data_rate[plane_id]);
1346
1347 if (DISPLAY_VER(display) < 11)
1348 skl_plane_calc_dbuf_bw(dbuf_bw, crtc, plane_id,
1349 ddb: &crtc_state->wm.skl.plane_ddb_y[plane_id],
1350 data_rate: crtc_state->data_rate[plane_id]);
1351 }
1352}
1353
1354/* "Maximum Data Buffer Bandwidth" */
1355static int
1356intel_bw_dbuf_min_cdclk(struct intel_display *display,
1357 const struct intel_bw_state *bw_state)
1358{
1359 unsigned int total_max_bw = 0;
1360 enum dbuf_slice slice;
1361
1362 for_each_dbuf_slice(display, slice) {
1363 int num_active_planes = 0;
1364 unsigned int max_bw = 0;
1365 enum pipe pipe;
1366
1367 /*
1368 * The arbiter can only really guarantee an
1369 * equal share of the total bw to each plane.
1370 */
1371 for_each_pipe(display, pipe) {
1372 const struct intel_dbuf_bw *dbuf_bw = &bw_state->dbuf_bw[pipe];
1373
1374 max_bw = max(dbuf_bw->max_bw[slice], max_bw);
1375 num_active_planes += hweight8(dbuf_bw->active_planes[slice]);
1376 }
1377 max_bw *= num_active_planes;
1378
1379 total_max_bw = max(total_max_bw, max_bw);
1380 }
1381
1382 return DIV_ROUND_UP(total_max_bw, 64);
1383}
1384
1385int intel_bw_min_cdclk(struct intel_display *display,
1386 const struct intel_bw_state *bw_state)
1387{
1388 enum pipe pipe;
1389 int min_cdclk;
1390
1391 min_cdclk = intel_bw_dbuf_min_cdclk(display, bw_state);
1392
1393 for_each_pipe(display, pipe)
1394 min_cdclk = max(min_cdclk,
1395 intel_bw_crtc_min_cdclk(display,
1396 bw_state->data_rate[pipe]));
1397
1398 return min_cdclk;
1399}
1400
1401int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1402 bool *need_cdclk_calc)
1403{
1404 struct intel_display *display = to_intel_display(state);
1405 struct intel_bw_state *new_bw_state = NULL;
1406 const struct intel_bw_state *old_bw_state = NULL;
1407 const struct intel_cdclk_state *cdclk_state;
1408 const struct intel_crtc_state *old_crtc_state;
1409 const struct intel_crtc_state *new_crtc_state;
1410 int old_min_cdclk, new_min_cdclk;
1411 struct intel_crtc *crtc;
1412 int i;
1413
1414 if (DISPLAY_VER(display) < 9)
1415 return 0;
1416
1417 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1418 new_crtc_state, i) {
1419 struct intel_dbuf_bw old_dbuf_bw, new_dbuf_bw;
1420
1421 skl_crtc_calc_dbuf_bw(dbuf_bw: &old_dbuf_bw, crtc_state: old_crtc_state);
1422 skl_crtc_calc_dbuf_bw(dbuf_bw: &new_dbuf_bw, crtc_state: new_crtc_state);
1423
1424 if (!intel_dbuf_bw_changed(display, old_dbuf_bw: &old_dbuf_bw, new_dbuf_bw: &new_dbuf_bw))
1425 continue;
1426
1427 new_bw_state = intel_atomic_get_bw_state(state);
1428 if (IS_ERR(ptr: new_bw_state))
1429 return PTR_ERR(ptr: new_bw_state);
1430
1431 old_bw_state = intel_atomic_get_old_bw_state(state);
1432
1433 new_bw_state->dbuf_bw[crtc->pipe] = new_dbuf_bw;
1434 }
1435
1436 if (!old_bw_state)
1437 return 0;
1438
1439 if (intel_bw_state_changed(display, old_bw_state, new_bw_state)) {
1440 int ret = intel_atomic_lock_global_state(obj_state: &new_bw_state->base);
1441 if (ret)
1442 return ret;
1443 }
1444
1445 old_min_cdclk = intel_bw_min_cdclk(display, bw_state: old_bw_state);
1446 new_min_cdclk = intel_bw_min_cdclk(display, bw_state: new_bw_state);
1447
1448 /*
1449 * No need to check against the cdclk state if
1450 * the min cdclk doesn't increase.
1451 *
1452 * Ie. we only ever increase the cdclk due to bandwidth
1453 * requirements. This can reduce back and forth
1454 * display blinking due to constant cdclk changes.
1455 */
1456 if (new_min_cdclk <= old_min_cdclk)
1457 return 0;
1458
1459 cdclk_state = intel_atomic_get_cdclk_state(state);
1460 if (IS_ERR(ptr: cdclk_state))
1461 return PTR_ERR(ptr: cdclk_state);
1462
1463 /*
1464 * No need to recalculate the cdclk state if
1465 * the min cdclk doesn't increase.
1466 *
1467 * Ie. we only ever increase the cdclk due to bandwidth
1468 * requirements. This can reduce back and forth
1469 * display blinking due to constant cdclk changes.
1470 */
1471 if (new_min_cdclk <= intel_cdclk_bw_min_cdclk(cdclk_state))
1472 return 0;
1473
1474 drm_dbg_kms(display->drm,
1475 "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1476 new_min_cdclk, intel_cdclk_bw_min_cdclk(cdclk_state));
1477 *need_cdclk_calc = true;
1478
1479 return 0;
1480}
1481
1482static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1483{
1484 struct intel_display *display = to_intel_display(state);
1485 const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1486 struct intel_crtc *crtc;
1487 int i;
1488
1489 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1490 new_crtc_state, i) {
1491 unsigned int old_data_rate =
1492 intel_bw_crtc_data_rate(crtc_state: old_crtc_state);
1493 unsigned int new_data_rate =
1494 intel_bw_crtc_data_rate(crtc_state: new_crtc_state);
1495 unsigned int old_active_planes =
1496 intel_bw_crtc_num_active_planes(crtc_state: old_crtc_state);
1497 unsigned int new_active_planes =
1498 intel_bw_crtc_num_active_planes(crtc_state: new_crtc_state);
1499 struct intel_bw_state *new_bw_state;
1500
1501 /*
1502 * Avoid locking the bw state when
1503 * nothing significant has changed.
1504 */
1505 if (old_data_rate == new_data_rate &&
1506 old_active_planes == new_active_planes)
1507 continue;
1508
1509 new_bw_state = intel_atomic_get_bw_state(state);
1510 if (IS_ERR(ptr: new_bw_state))
1511 return PTR_ERR(ptr: new_bw_state);
1512
1513 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1514 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1515
1516 *changed = true;
1517
1518 drm_dbg_kms(display->drm,
1519 "[CRTC:%d:%s] data rate %u num active planes %u\n",
1520 crtc->base.base.id, crtc->base.name,
1521 new_bw_state->data_rate[crtc->pipe],
1522 new_bw_state->num_active_planes[crtc->pipe]);
1523 }
1524
1525 return 0;
1526}
1527
1528static int intel_bw_modeset_checks(struct intel_atomic_state *state)
1529{
1530 struct intel_display *display = to_intel_display(state);
1531 const struct intel_bw_state *old_bw_state;
1532 struct intel_bw_state *new_bw_state;
1533
1534 if (DISPLAY_VER(display) < 9)
1535 return 0;
1536
1537 new_bw_state = intel_atomic_get_bw_state(state);
1538 if (IS_ERR(ptr: new_bw_state))
1539 return PTR_ERR(ptr: new_bw_state);
1540
1541 old_bw_state = intel_atomic_get_old_bw_state(state);
1542
1543 new_bw_state->active_pipes =
1544 intel_calc_active_pipes(state, active_pipes: old_bw_state->active_pipes);
1545
1546 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
1547 int ret;
1548
1549 ret = intel_atomic_lock_global_state(obj_state: &new_bw_state->base);
1550 if (ret)
1551 return ret;
1552 }
1553
1554 return 0;
1555}
1556
1557static int intel_bw_check_sagv_mask(struct intel_atomic_state *state)
1558{
1559 struct intel_display *display = to_intel_display(state);
1560 const struct intel_crtc_state *old_crtc_state;
1561 const struct intel_crtc_state *new_crtc_state;
1562 const struct intel_bw_state *old_bw_state = NULL;
1563 struct intel_bw_state *new_bw_state = NULL;
1564 struct intel_crtc *crtc;
1565 int ret, i;
1566
1567 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1568 new_crtc_state, i) {
1569 if (intel_crtc_can_enable_sagv(crtc_state: old_crtc_state) ==
1570 intel_crtc_can_enable_sagv(crtc_state: new_crtc_state))
1571 continue;
1572
1573 new_bw_state = intel_atomic_get_bw_state(state);
1574 if (IS_ERR(ptr: new_bw_state))
1575 return PTR_ERR(ptr: new_bw_state);
1576
1577 old_bw_state = intel_atomic_get_old_bw_state(state);
1578
1579 if (intel_crtc_can_enable_sagv(crtc_state: new_crtc_state))
1580 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
1581 else
1582 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
1583 }
1584
1585 if (!new_bw_state)
1586 return 0;
1587
1588 if (intel_bw_can_enable_sagv(display, bw_state: new_bw_state) !=
1589 intel_bw_can_enable_sagv(display, bw_state: old_bw_state)) {
1590 ret = intel_atomic_serialize_global_state(obj_state: &new_bw_state->base);
1591 if (ret)
1592 return ret;
1593 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
1594 ret = intel_atomic_lock_global_state(obj_state: &new_bw_state->base);
1595 if (ret)
1596 return ret;
1597 }
1598
1599 return 0;
1600}
1601
1602int intel_bw_atomic_check(struct intel_atomic_state *state, bool any_ms)
1603{
1604 struct intel_display *display = to_intel_display(state);
1605 bool changed = false;
1606 struct intel_bw_state *new_bw_state;
1607 const struct intel_bw_state *old_bw_state;
1608 int ret;
1609
1610 if (DISPLAY_VER(display) < 9)
1611 return 0;
1612
1613 if (any_ms) {
1614 ret = intel_bw_modeset_checks(state);
1615 if (ret)
1616 return ret;
1617 }
1618
1619 ret = intel_bw_check_sagv_mask(state);
1620 if (ret)
1621 return ret;
1622
1623 /* FIXME earlier gens need some checks too */
1624 if (DISPLAY_VER(display) < 11)
1625 return 0;
1626
1627 ret = intel_bw_check_data_rate(state, changed: &changed);
1628 if (ret)
1629 return ret;
1630
1631 old_bw_state = intel_atomic_get_old_bw_state(state);
1632 new_bw_state = intel_atomic_get_new_bw_state(state);
1633
1634 if (new_bw_state &&
1635 intel_bw_can_enable_sagv(display, bw_state: old_bw_state) !=
1636 intel_bw_can_enable_sagv(display, bw_state: new_bw_state))
1637 changed = true;
1638
1639 /*
1640 * If none of our inputs (data rates, number of active
1641 * planes, SAGV yes/no) changed then nothing to do here.
1642 */
1643 if (!changed)
1644 return 0;
1645
1646 ret = intel_bw_check_qgv_points(display, old_bw_state, new_bw_state);
1647 if (ret)
1648 return ret;
1649
1650 return 0;
1651}
1652
1653static void intel_bw_crtc_update(struct intel_bw_state *bw_state,
1654 const struct intel_crtc_state *crtc_state)
1655{
1656 struct intel_display *display = to_intel_display(crtc_state);
1657 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1658
1659 bw_state->data_rate[crtc->pipe] =
1660 intel_bw_crtc_data_rate(crtc_state);
1661 bw_state->num_active_planes[crtc->pipe] =
1662 intel_bw_crtc_num_active_planes(crtc_state);
1663
1664 drm_dbg_kms(display->drm, "pipe %c data rate %u num active planes %u\n",
1665 pipe_name(crtc->pipe),
1666 bw_state->data_rate[crtc->pipe],
1667 bw_state->num_active_planes[crtc->pipe]);
1668}
1669
1670void intel_bw_update_hw_state(struct intel_display *display)
1671{
1672 struct intel_bw_state *bw_state =
1673 to_intel_bw_state(obj_state: display->bw.obj.state);
1674 struct intel_crtc *crtc;
1675
1676 if (DISPLAY_VER(display) < 9)
1677 return;
1678
1679 bw_state->active_pipes = 0;
1680 bw_state->pipe_sagv_reject = 0;
1681
1682 for_each_intel_crtc(display->drm, crtc) {
1683 const struct intel_crtc_state *crtc_state =
1684 to_intel_crtc_state(crtc->base.state);
1685 enum pipe pipe = crtc->pipe;
1686
1687 if (crtc_state->hw.active)
1688 bw_state->active_pipes |= BIT(pipe);
1689
1690 if (DISPLAY_VER(display) >= 11)
1691 intel_bw_crtc_update(bw_state, crtc_state);
1692
1693 skl_crtc_calc_dbuf_bw(dbuf_bw: &bw_state->dbuf_bw[pipe], crtc_state);
1694
1695 /* initially SAGV has been forced off */
1696 bw_state->pipe_sagv_reject |= BIT(pipe);
1697 }
1698}
1699
1700void intel_bw_crtc_disable_noatomic(struct intel_crtc *crtc)
1701{
1702 struct intel_display *display = to_intel_display(crtc);
1703 struct intel_bw_state *bw_state =
1704 to_intel_bw_state(obj_state: display->bw.obj.state);
1705 enum pipe pipe = crtc->pipe;
1706
1707 if (DISPLAY_VER(display) < 9)
1708 return;
1709
1710 bw_state->data_rate[pipe] = 0;
1711 bw_state->num_active_planes[pipe] = 0;
1712 memset(s: &bw_state->dbuf_bw[pipe], c: 0, n: sizeof(bw_state->dbuf_bw[pipe]));
1713}
1714
1715static struct intel_global_state *
1716intel_bw_duplicate_state(struct intel_global_obj *obj)
1717{
1718 struct intel_bw_state *state;
1719
1720 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1721 if (!state)
1722 return NULL;
1723
1724 return &state->base;
1725}
1726
1727static void intel_bw_destroy_state(struct intel_global_obj *obj,
1728 struct intel_global_state *state)
1729{
1730 kfree(objp: state);
1731}
1732
1733static const struct intel_global_state_funcs intel_bw_funcs = {
1734 .atomic_duplicate_state = intel_bw_duplicate_state,
1735 .atomic_destroy_state = intel_bw_destroy_state,
1736};
1737
1738int intel_bw_init(struct intel_display *display)
1739{
1740 struct intel_bw_state *state;
1741
1742 state = kzalloc(sizeof(*state), GFP_KERNEL);
1743 if (!state)
1744 return -ENOMEM;
1745
1746 intel_atomic_global_obj_init(display, obj: &display->bw.obj,
1747 state: &state->base, funcs: &intel_bw_funcs);
1748
1749 /*
1750 * Limit this only if we have SAGV. And for Display version 14 onwards
1751 * sagv is handled though pmdemand requests
1752 */
1753 if (intel_has_sagv(display) && IS_DISPLAY_VER(display, 11, 13))
1754 icl_force_disable_sagv(display, bw_state: state);
1755
1756 return 0;
1757}
1758
1759bool intel_bw_pmdemand_needs_update(struct intel_atomic_state *state)
1760{
1761 const struct intel_bw_state *new_bw_state, *old_bw_state;
1762
1763 new_bw_state = intel_atomic_get_new_bw_state(state);
1764 old_bw_state = intel_atomic_get_old_bw_state(state);
1765
1766 if (new_bw_state &&
1767 new_bw_state->qgv_point_peakbw != old_bw_state->qgv_point_peakbw)
1768 return true;
1769
1770 return false;
1771}
1772
1773bool intel_bw_can_enable_sagv(struct intel_display *display,
1774 const struct intel_bw_state *bw_state)
1775{
1776 if (DISPLAY_VER(display) < 11 &&
1777 bw_state->active_pipes && !is_power_of_2(n: bw_state->active_pipes))
1778 return false;
1779
1780 return bw_state->pipe_sagv_reject == 0;
1781}
1782
1783int intel_bw_qgv_point_peakbw(const struct intel_bw_state *bw_state)
1784{
1785 return bw_state->qgv_point_peakbw;
1786}
1787