1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include <linux/debugfs.h>
7
8#include <drm/drm_blend.h>
9#include <drm/drm_print.h>
10
11#include "soc/intel_dram.h"
12#include "i915_reg.h"
13#include "i915_utils.h"
14#include "i9xx_wm.h"
15#include "intel_atomic.h"
16#include "intel_bw.h"
17#include "intel_cdclk.h"
18#include "intel_crtc.h"
19#include "intel_cursor_regs.h"
20#include "intel_de.h"
21#include "intel_display.h"
22#include "intel_display_power.h"
23#include "intel_display_regs.h"
24#include "intel_display_rpm.h"
25#include "intel_display_types.h"
26#include "intel_fb.h"
27#include "intel_fixed.h"
28#include "intel_flipq.h"
29#include "intel_pcode.h"
30#include "intel_plane.h"
31#include "intel_wm.h"
32#include "skl_universal_plane_regs.h"
33#include "skl_watermark.h"
34#include "skl_watermark_regs.h"
35
36struct intel_dbuf_state {
37 struct intel_global_state base;
38
39 struct skl_ddb_entry ddb[I915_MAX_PIPES];
40 unsigned int weight[I915_MAX_PIPES];
41 u8 slices[I915_MAX_PIPES];
42 u8 enabled_slices;
43 u8 active_pipes;
44 u8 mdclk_cdclk_ratio;
45 bool joined_mbus;
46};
47
48#define to_intel_dbuf_state(global_state) \
49 container_of_const((global_state), struct intel_dbuf_state, base)
50
51#define intel_atomic_get_old_dbuf_state(state) \
52 to_intel_dbuf_state(intel_atomic_get_old_global_obj_state(state, &to_intel_display(state)->dbuf.obj))
53#define intel_atomic_get_new_dbuf_state(state) \
54 to_intel_dbuf_state(intel_atomic_get_new_global_obj_state(state, &to_intel_display(state)->dbuf.obj))
55
56static void skl_sagv_disable(struct intel_display *display);
57
58/* Stores plane specific WM parameters */
59struct skl_wm_params {
60 bool x_tiled, y_tiled;
61 bool rc_surface;
62 bool is_planar;
63 u32 width;
64 u8 cpp;
65 u32 plane_pixel_rate;
66 u32 y_min_scanlines;
67 u32 plane_bytes_per_line;
68 uint_fixed_16_16_t plane_blocks_per_line;
69 uint_fixed_16_16_t y_tile_minimum;
70 u32 linetime_us;
71 u32 dbuf_block_size;
72};
73
74u8 intel_enabled_dbuf_slices_mask(struct intel_display *display)
75{
76 u8 enabled_slices = 0;
77 enum dbuf_slice slice;
78
79 for_each_dbuf_slice(display, slice) {
80 if (intel_de_read(display, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
81 enabled_slices |= BIT(slice);
82 }
83
84 return enabled_slices;
85}
86
87/*
88 * FIXME: We still don't have the proper code detect if we need to apply the WA,
89 * so assume we'll always need it in order to avoid underruns.
90 */
91static bool skl_needs_memory_bw_wa(struct intel_display *display)
92{
93 return DISPLAY_VER(display) == 9;
94}
95
96bool
97intel_has_sagv(struct intel_display *display)
98{
99 return HAS_SAGV(display) && display->sagv.status != I915_SAGV_NOT_CONTROLLED;
100}
101
102static u32
103intel_sagv_block_time(struct intel_display *display)
104{
105 if (DISPLAY_VER(display) >= 14) {
106 u32 val;
107
108 val = intel_de_read(display, MTL_LATENCY_SAGV);
109
110 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
111 } else if (DISPLAY_VER(display) >= 12) {
112 u32 val = 0;
113 int ret;
114
115 ret = intel_pcode_read(drm: display->drm,
116 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
117 val: &val, NULL);
118 if (ret) {
119 drm_dbg_kms(display->drm, "Couldn't read SAGV block time!\n");
120 return 0;
121 }
122
123 return val;
124 } else if (DISPLAY_VER(display) == 11) {
125 return 10;
126 } else if (HAS_SAGV(display)) {
127 return 30;
128 } else {
129 return 0;
130 }
131}
132
133static void intel_sagv_init(struct intel_display *display)
134{
135 if (!HAS_SAGV(display))
136 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
137
138 /*
139 * Probe to see if we have working SAGV control.
140 * For icl+ this was already determined by intel_bw_init_hw().
141 */
142 if (DISPLAY_VER(display) < 11)
143 skl_sagv_disable(display);
144
145 drm_WARN_ON(display->drm, display->sagv.status == I915_SAGV_UNKNOWN);
146
147 display->sagv.block_time_us = intel_sagv_block_time(display);
148
149 drm_dbg_kms(display->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
150 str_yes_no(intel_has_sagv(display)), display->sagv.block_time_us);
151
152 /* avoid overflow when adding with wm0 latency/etc. */
153 if (drm_WARN(display->drm, display->sagv.block_time_us > U16_MAX,
154 "Excessive SAGV block time %u, ignoring\n",
155 display->sagv.block_time_us))
156 display->sagv.block_time_us = 0;
157
158 if (!intel_has_sagv(display))
159 display->sagv.block_time_us = 0;
160}
161
162/*
163 * SAGV dynamically adjusts the system agent voltage and clock frequencies
164 * depending on power and performance requirements. The display engine access
165 * to system memory is blocked during the adjustment time. Because of the
166 * blocking time, having this enabled can cause full system hangs and/or pipe
167 * underruns if we don't meet all of the following requirements:
168 *
169 * - <= 1 pipe enabled
170 * - All planes can enable watermarks for latencies >= SAGV engine block time
171 * - We're not using an interlaced display configuration
172 */
173static void skl_sagv_enable(struct intel_display *display)
174{
175 int ret;
176
177 if (!intel_has_sagv(display))
178 return;
179
180 if (display->sagv.status == I915_SAGV_ENABLED)
181 return;
182
183 drm_dbg_kms(display->drm, "Enabling SAGV\n");
184 ret = intel_pcode_write(display->drm, GEN9_PCODE_SAGV_CONTROL,
185 GEN9_SAGV_ENABLE);
186
187 /* We don't need to wait for SAGV when enabling */
188
189 /*
190 * Some skl systems, pre-release machines in particular,
191 * don't actually have SAGV.
192 */
193 if (display->platform.skylake && ret == -ENXIO) {
194 drm_dbg(display->drm, "No SAGV found on system, ignoring\n");
195 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
196 return;
197 } else if (ret < 0) {
198 drm_err(display->drm, "Failed to enable SAGV\n");
199 return;
200 }
201
202 display->sagv.status = I915_SAGV_ENABLED;
203}
204
205static void skl_sagv_disable(struct intel_display *display)
206{
207 int ret;
208
209 if (!intel_has_sagv(display))
210 return;
211
212 if (display->sagv.status == I915_SAGV_DISABLED)
213 return;
214
215 drm_dbg_kms(display->drm, "Disabling SAGV\n");
216 /* bspec says to keep retrying for at least 1 ms */
217 ret = intel_pcode_request(drm: display->drm, GEN9_PCODE_SAGV_CONTROL,
218 GEN9_SAGV_DISABLE,
219 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, timeout_base_ms: 1);
220 /*
221 * Some skl systems, pre-release machines in particular,
222 * don't actually have SAGV.
223 */
224 if (display->platform.skylake && ret == -ENXIO) {
225 drm_dbg(display->drm, "No SAGV found on system, ignoring\n");
226 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
227 return;
228 } else if (ret < 0) {
229 drm_err(display->drm, "Failed to disable SAGV (%d)\n", ret);
230 return;
231 }
232
233 display->sagv.status = I915_SAGV_DISABLED;
234}
235
236static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
237{
238 struct intel_display *display = to_intel_display(state);
239 const struct intel_bw_state *new_bw_state =
240 intel_atomic_get_new_bw_state(state);
241
242 if (!new_bw_state)
243 return;
244
245 if (!intel_bw_can_enable_sagv(display, bw_state: new_bw_state))
246 skl_sagv_disable(display);
247}
248
249static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
250{
251 struct intel_display *display = to_intel_display(state);
252 const struct intel_bw_state *new_bw_state =
253 intel_atomic_get_new_bw_state(state);
254
255 if (!new_bw_state)
256 return;
257
258 if (intel_bw_can_enable_sagv(display, bw_state: new_bw_state))
259 skl_sagv_enable(display);
260}
261
262void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
263{
264 struct intel_display *display = to_intel_display(state);
265
266 /*
267 * Just return if we can't control SAGV or don't have it.
268 * This is different from situation when we have SAGV but just can't
269 * afford it due to DBuf limitation - in case if SAGV is completely
270 * disabled in a BIOS, we are not even allowed to send a PCode request,
271 * as it will throw an error. So have to check it here.
272 */
273 if (!intel_has_sagv(display))
274 return;
275
276 if (DISPLAY_VER(display) >= 11)
277 icl_sagv_pre_plane_update(state);
278 else
279 skl_sagv_pre_plane_update(state);
280}
281
282void intel_sagv_post_plane_update(struct intel_atomic_state *state)
283{
284 struct intel_display *display = to_intel_display(state);
285
286 /*
287 * Just return if we can't control SAGV or don't have it.
288 * This is different from situation when we have SAGV but just can't
289 * afford it due to DBuf limitation - in case if SAGV is completely
290 * disabled in a BIOS, we are not even allowed to send a PCode request,
291 * as it will throw an error. So have to check it here.
292 */
293 if (!intel_has_sagv(display))
294 return;
295
296 if (DISPLAY_VER(display) >= 11)
297 icl_sagv_post_plane_update(state);
298 else
299 skl_sagv_post_plane_update(state);
300}
301
302static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
303{
304 struct intel_display *display = to_intel_display(crtc_state);
305 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
306 enum plane_id plane_id;
307 int max_level = INT_MAX;
308
309 if (!intel_has_sagv(display))
310 return false;
311
312 if (!crtc_state->hw.active)
313 return true;
314
315 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
316 return false;
317
318 for_each_plane_id_on_crtc(crtc, plane_id) {
319 const struct skl_plane_wm *wm =
320 &crtc_state->wm.skl.optimal.planes[plane_id];
321 int level;
322
323 /* Skip this plane if it's not enabled */
324 if (!wm->wm[0].enable)
325 continue;
326
327 /* Find the highest enabled wm level for this plane */
328 for (level = display->wm.num_levels - 1;
329 !wm->wm[level].enable; --level)
330 { }
331
332 /* Highest common enabled wm level for all planes */
333 max_level = min(level, max_level);
334 }
335
336 /* No enabled planes? */
337 if (max_level == INT_MAX)
338 return true;
339
340 for_each_plane_id_on_crtc(crtc, plane_id) {
341 const struct skl_plane_wm *wm =
342 &crtc_state->wm.skl.optimal.planes[plane_id];
343
344 /*
345 * All enabled planes must have enabled a common wm level that
346 * can tolerate memory latencies higher than sagv_block_time_us
347 */
348 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
349 return false;
350 }
351
352 return true;
353}
354
355static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
356{
357 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
358 enum plane_id plane_id;
359
360 if (!crtc_state->hw.active)
361 return true;
362
363 for_each_plane_id_on_crtc(crtc, plane_id) {
364 const struct skl_plane_wm *wm =
365 &crtc_state->wm.skl.optimal.planes[plane_id];
366
367 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
368 return false;
369 }
370
371 return true;
372}
373
374bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
375{
376 struct intel_display *display = to_intel_display(crtc_state);
377
378 if (!display->params.enable_sagv)
379 return false;
380
381 /*
382 * SAGV is initially forced off because its current
383 * state can't be queried from pcode. Allow SAGV to
384 * be enabled upon the first real commit.
385 */
386 if (crtc_state->inherited)
387 return false;
388
389 if (DISPLAY_VER(display) >= 12)
390 return tgl_crtc_can_enable_sagv(crtc_state);
391 else
392 return skl_crtc_can_enable_sagv(crtc_state);
393}
394
395static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
396 u16 start, u16 end)
397{
398 entry->start = start;
399 entry->end = end;
400
401 return end;
402}
403
404static int intel_dbuf_slice_size(struct intel_display *display)
405{
406 return DISPLAY_INFO(display)->dbuf.size /
407 hweight8(DISPLAY_INFO(display)->dbuf.slice_mask);
408}
409
410static void
411skl_ddb_entry_for_slices(struct intel_display *display, u8 slice_mask,
412 struct skl_ddb_entry *ddb)
413{
414 int slice_size = intel_dbuf_slice_size(display);
415
416 if (!slice_mask) {
417 ddb->start = 0;
418 ddb->end = 0;
419 return;
420 }
421
422 ddb->start = (ffs(slice_mask) - 1) * slice_size;
423 ddb->end = fls(x: slice_mask) * slice_size;
424
425 WARN_ON(ddb->start >= ddb->end);
426 WARN_ON(ddb->end > DISPLAY_INFO(display)->dbuf.size);
427}
428
429static unsigned int mbus_ddb_offset(struct intel_display *display, u8 slice_mask)
430{
431 struct skl_ddb_entry ddb;
432
433 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
434 slice_mask = BIT(DBUF_S1);
435 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
436 slice_mask = BIT(DBUF_S3);
437
438 skl_ddb_entry_for_slices(display, slice_mask, ddb: &ddb);
439
440 return ddb.start;
441}
442
443u32 skl_ddb_dbuf_slice_mask(struct intel_display *display,
444 const struct skl_ddb_entry *entry)
445{
446 int slice_size = intel_dbuf_slice_size(display);
447 enum dbuf_slice start_slice, end_slice;
448 u8 slice_mask = 0;
449
450 if (!skl_ddb_entry_size(entry))
451 return 0;
452
453 start_slice = entry->start / slice_size;
454 end_slice = (entry->end - 1) / slice_size;
455
456 /*
457 * Per plane DDB entry can in a really worst case be on multiple slices
458 * but single entry is anyway contiguous.
459 */
460 while (start_slice <= end_slice) {
461 slice_mask |= BIT(start_slice);
462 start_slice++;
463 }
464
465 return slice_mask;
466}
467
468static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
469{
470 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
471 int hdisplay, vdisplay;
472
473 if (!crtc_state->hw.active)
474 return 0;
475
476 /*
477 * Watermark/ddb requirement highly depends upon width of the
478 * framebuffer, So instead of allocating DDB equally among pipes
479 * distribute DDB based on resolution/width of the display.
480 */
481 drm_mode_get_hv_timing(mode: pipe_mode, hdisplay: &hdisplay, vdisplay: &vdisplay);
482
483 return hdisplay;
484}
485
486static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
487 enum pipe for_pipe,
488 unsigned int *weight_start,
489 unsigned int *weight_end,
490 unsigned int *weight_total)
491{
492 struct intel_display *display = to_intel_display(dbuf_state->base.state->base.dev);
493 enum pipe pipe;
494
495 *weight_start = 0;
496 *weight_end = 0;
497 *weight_total = 0;
498
499 for_each_pipe(display, pipe) {
500 int weight = dbuf_state->weight[pipe];
501
502 /*
503 * Do not account pipes using other slice sets
504 * luckily as of current BSpec slice sets do not partially
505 * intersect(pipes share either same one slice or same slice set
506 * i.e no partial intersection), so it is enough to check for
507 * equality for now.
508 */
509 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
510 continue;
511
512 *weight_total += weight;
513 if (pipe < for_pipe) {
514 *weight_start += weight;
515 *weight_end += weight;
516 } else if (pipe == for_pipe) {
517 *weight_end += weight;
518 }
519 }
520}
521
522static int
523skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
524{
525 struct intel_display *display = to_intel_display(crtc);
526 unsigned int weight_total, weight_start, weight_end;
527 const struct intel_dbuf_state *old_dbuf_state =
528 intel_atomic_get_old_dbuf_state(state);
529 struct intel_dbuf_state *new_dbuf_state =
530 intel_atomic_get_new_dbuf_state(state);
531 struct intel_crtc_state *crtc_state;
532 struct skl_ddb_entry ddb_slices;
533 enum pipe pipe = crtc->pipe;
534 unsigned int mbus_offset = 0;
535 u32 ddb_range_size;
536 u32 dbuf_slice_mask;
537 u32 start, end;
538 int ret;
539
540 if (new_dbuf_state->weight[pipe] == 0) {
541 skl_ddb_entry_init(entry: &new_dbuf_state->ddb[pipe], start: 0, end: 0);
542 goto out;
543 }
544
545 dbuf_slice_mask = new_dbuf_state->slices[pipe];
546
547 skl_ddb_entry_for_slices(display, slice_mask: dbuf_slice_mask, ddb: &ddb_slices);
548 mbus_offset = mbus_ddb_offset(display, slice_mask: dbuf_slice_mask);
549 ddb_range_size = skl_ddb_entry_size(entry: &ddb_slices);
550
551 intel_crtc_dbuf_weights(dbuf_state: new_dbuf_state, for_pipe: pipe,
552 weight_start: &weight_start, weight_end: &weight_end, weight_total: &weight_total);
553
554 start = ddb_range_size * weight_start / weight_total;
555 end = ddb_range_size * weight_end / weight_total;
556
557 skl_ddb_entry_init(entry: &new_dbuf_state->ddb[pipe],
558 start: ddb_slices.start - mbus_offset + start,
559 end: ddb_slices.start - mbus_offset + end);
560
561out:
562 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
563 skl_ddb_entry_equal(e1: &old_dbuf_state->ddb[pipe],
564 e2: &new_dbuf_state->ddb[pipe]))
565 return 0;
566
567 ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
568 if (ret)
569 return ret;
570
571 crtc_state = intel_atomic_get_crtc_state(state: &state->base, crtc);
572 if (IS_ERR(ptr: crtc_state))
573 return PTR_ERR(ptr: crtc_state);
574
575 /*
576 * Used for checking overlaps, so we need absolute
577 * offsets instead of MBUS relative offsets.
578 */
579 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
580 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
581
582 drm_dbg_kms(display->drm,
583 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
584 crtc->base.base.id, crtc->base.name,
585 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
586 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
587 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
588 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
589
590 return 0;
591}
592
593static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
594 int width, const struct drm_format_info *format,
595 u64 modifier, unsigned int rotation,
596 u32 plane_pixel_rate, struct skl_wm_params *wp,
597 int color_plane, unsigned int pan_x);
598
599static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
600 struct intel_plane *plane,
601 int level,
602 unsigned int latency,
603 const struct skl_wm_params *wp,
604 const struct skl_wm_level *result_prev,
605 struct skl_wm_level *result /* out */);
606
607static unsigned int skl_wm_latency(struct intel_display *display, int level,
608 const struct skl_wm_params *wp)
609{
610 unsigned int latency = display->wm.skl_latency[level];
611
612 if (latency == 0)
613 return 0;
614
615 /*
616 * WaIncreaseLatencyIPCEnabled: kbl,cfl
617 * Display WA #1141: kbl,cfl
618 */
619 if ((display->platform.kabylake || display->platform.coffeelake ||
620 display->platform.cometlake) && skl_watermark_ipc_enabled(display))
621 latency += 4;
622
623 if (skl_needs_memory_bw_wa(display) && wp && wp->x_tiled)
624 latency += 15;
625
626 return latency;
627}
628
629static unsigned int
630skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
631 int num_active)
632{
633 struct intel_display *display = to_intel_display(crtc_state);
634 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
635 struct skl_wm_level wm = {};
636 int ret, min_ddb_alloc = 0;
637 struct skl_wm_params wp;
638 int level;
639
640 ret = skl_compute_wm_params(crtc_state, width: 256,
641 format: drm_format_info(DRM_FORMAT_ARGB8888),
642 DRM_FORMAT_MOD_LINEAR,
643 DRM_MODE_ROTATE_0,
644 plane_pixel_rate: crtc_state->pixel_rate, wp: &wp, color_plane: 0, pan_x: 0);
645 drm_WARN_ON(display->drm, ret);
646
647 for (level = 0; level < display->wm.num_levels; level++) {
648 unsigned int latency = skl_wm_latency(display, level, wp: &wp);
649
650 skl_compute_plane_wm(crtc_state, plane, level, latency, wp: &wp, result_prev: &wm, result: &wm);
651 if (wm.min_ddb_alloc == U16_MAX)
652 break;
653
654 min_ddb_alloc = wm.min_ddb_alloc;
655 }
656
657 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
658}
659
660static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
661{
662 skl_ddb_entry_init(entry,
663 REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
664 REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
665 if (entry->end)
666 entry->end++;
667}
668
669static void
670skl_ddb_get_hw_plane_state(struct intel_display *display,
671 const enum pipe pipe,
672 const enum plane_id plane_id,
673 struct skl_ddb_entry *ddb,
674 struct skl_ddb_entry *ddb_y,
675 u16 *min_ddb, u16 *interim_ddb)
676{
677 u32 val;
678
679 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
680 if (plane_id == PLANE_CURSOR) {
681 val = intel_de_read(display, CUR_BUF_CFG(pipe));
682 skl_ddb_entry_init_from_hw(entry: ddb, reg: val);
683 return;
684 }
685
686 val = intel_de_read(display, PLANE_BUF_CFG(pipe, plane_id));
687 skl_ddb_entry_init_from_hw(entry: ddb, reg: val);
688
689 if (DISPLAY_VER(display) >= 30) {
690 val = intel_de_read(display, PLANE_MIN_BUF_CFG(pipe, plane_id));
691
692 *min_ddb = REG_FIELD_GET(PLANE_MIN_DBUF_BLOCKS_MASK, val);
693 *interim_ddb = REG_FIELD_GET(PLANE_INTERIM_DBUF_BLOCKS_MASK, val);
694 }
695
696 if (DISPLAY_VER(display) >= 11)
697 return;
698
699 val = intel_de_read(display, PLANE_NV12_BUF_CFG(pipe, plane_id));
700 skl_ddb_entry_init_from_hw(entry: ddb_y, reg: val);
701}
702
703static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
704 struct skl_ddb_entry *ddb,
705 struct skl_ddb_entry *ddb_y,
706 u16 *min_ddb, u16 *interim_ddb)
707{
708 struct intel_display *display = to_intel_display(crtc);
709 enum intel_display_power_domain power_domain;
710 enum pipe pipe = crtc->pipe;
711 intel_wakeref_t wakeref;
712 enum plane_id plane_id;
713
714 power_domain = POWER_DOMAIN_PIPE(pipe);
715 wakeref = intel_display_power_get_if_enabled(display, domain: power_domain);
716 if (!wakeref)
717 return;
718
719 for_each_plane_id_on_crtc(crtc, plane_id)
720 skl_ddb_get_hw_plane_state(display, pipe,
721 plane_id,
722 ddb: &ddb[plane_id],
723 ddb_y: &ddb_y[plane_id],
724 min_ddb: &min_ddb[plane_id],
725 interim_ddb: &interim_ddb[plane_id]);
726
727 intel_display_power_put(display, domain: power_domain, wakeref);
728}
729
730struct dbuf_slice_conf_entry {
731 u8 active_pipes;
732 u8 dbuf_mask[I915_MAX_PIPES];
733 bool join_mbus;
734};
735
736/*
737 * Table taken from Bspec 12716
738 * Pipes do have some preferred DBuf slice affinity,
739 * plus there are some hardcoded requirements on how
740 * those should be distributed for multipipe scenarios.
741 * For more DBuf slices algorithm can get even more messy
742 * and less readable, so decided to use a table almost
743 * as is from BSpec itself - that way it is at least easier
744 * to compare, change and check.
745 */
746static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
747/* Autogenerated with igt/tools/intel_dbuf_map tool: */
748{
749 {
750 .active_pipes = BIT(PIPE_A),
751 .dbuf_mask = {
752 [PIPE_A] = BIT(DBUF_S1),
753 },
754 },
755 {
756 .active_pipes = BIT(PIPE_B),
757 .dbuf_mask = {
758 [PIPE_B] = BIT(DBUF_S1),
759 },
760 },
761 {
762 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
763 .dbuf_mask = {
764 [PIPE_A] = BIT(DBUF_S1),
765 [PIPE_B] = BIT(DBUF_S2),
766 },
767 },
768 {
769 .active_pipes = BIT(PIPE_C),
770 .dbuf_mask = {
771 [PIPE_C] = BIT(DBUF_S2),
772 },
773 },
774 {
775 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
776 .dbuf_mask = {
777 [PIPE_A] = BIT(DBUF_S1),
778 [PIPE_C] = BIT(DBUF_S2),
779 },
780 },
781 {
782 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
783 .dbuf_mask = {
784 [PIPE_B] = BIT(DBUF_S1),
785 [PIPE_C] = BIT(DBUF_S2),
786 },
787 },
788 {
789 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
790 .dbuf_mask = {
791 [PIPE_A] = BIT(DBUF_S1),
792 [PIPE_B] = BIT(DBUF_S1),
793 [PIPE_C] = BIT(DBUF_S2),
794 },
795 },
796 {}
797};
798
799/*
800 * Table taken from Bspec 49255
801 * Pipes do have some preferred DBuf slice affinity,
802 * plus there are some hardcoded requirements on how
803 * those should be distributed for multipipe scenarios.
804 * For more DBuf slices algorithm can get even more messy
805 * and less readable, so decided to use a table almost
806 * as is from BSpec itself - that way it is at least easier
807 * to compare, change and check.
808 */
809static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
810/* Autogenerated with igt/tools/intel_dbuf_map tool: */
811{
812 {
813 .active_pipes = BIT(PIPE_A),
814 .dbuf_mask = {
815 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
816 },
817 },
818 {
819 .active_pipes = BIT(PIPE_B),
820 .dbuf_mask = {
821 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
822 },
823 },
824 {
825 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
826 .dbuf_mask = {
827 [PIPE_A] = BIT(DBUF_S2),
828 [PIPE_B] = BIT(DBUF_S1),
829 },
830 },
831 {
832 .active_pipes = BIT(PIPE_C),
833 .dbuf_mask = {
834 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
835 },
836 },
837 {
838 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
839 .dbuf_mask = {
840 [PIPE_A] = BIT(DBUF_S1),
841 [PIPE_C] = BIT(DBUF_S2),
842 },
843 },
844 {
845 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
846 .dbuf_mask = {
847 [PIPE_B] = BIT(DBUF_S1),
848 [PIPE_C] = BIT(DBUF_S2),
849 },
850 },
851 {
852 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
853 .dbuf_mask = {
854 [PIPE_A] = BIT(DBUF_S1),
855 [PIPE_B] = BIT(DBUF_S1),
856 [PIPE_C] = BIT(DBUF_S2),
857 },
858 },
859 {
860 .active_pipes = BIT(PIPE_D),
861 .dbuf_mask = {
862 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
863 },
864 },
865 {
866 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
867 .dbuf_mask = {
868 [PIPE_A] = BIT(DBUF_S1),
869 [PIPE_D] = BIT(DBUF_S2),
870 },
871 },
872 {
873 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
874 .dbuf_mask = {
875 [PIPE_B] = BIT(DBUF_S1),
876 [PIPE_D] = BIT(DBUF_S2),
877 },
878 },
879 {
880 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
881 .dbuf_mask = {
882 [PIPE_A] = BIT(DBUF_S1),
883 [PIPE_B] = BIT(DBUF_S1),
884 [PIPE_D] = BIT(DBUF_S2),
885 },
886 },
887 {
888 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
889 .dbuf_mask = {
890 [PIPE_C] = BIT(DBUF_S1),
891 [PIPE_D] = BIT(DBUF_S2),
892 },
893 },
894 {
895 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
896 .dbuf_mask = {
897 [PIPE_A] = BIT(DBUF_S1),
898 [PIPE_C] = BIT(DBUF_S2),
899 [PIPE_D] = BIT(DBUF_S2),
900 },
901 },
902 {
903 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
904 .dbuf_mask = {
905 [PIPE_B] = BIT(DBUF_S1),
906 [PIPE_C] = BIT(DBUF_S2),
907 [PIPE_D] = BIT(DBUF_S2),
908 },
909 },
910 {
911 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
912 .dbuf_mask = {
913 [PIPE_A] = BIT(DBUF_S1),
914 [PIPE_B] = BIT(DBUF_S1),
915 [PIPE_C] = BIT(DBUF_S2),
916 [PIPE_D] = BIT(DBUF_S2),
917 },
918 },
919 {}
920};
921
922static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
923 {
924 .active_pipes = BIT(PIPE_A),
925 .dbuf_mask = {
926 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
927 },
928 },
929 {
930 .active_pipes = BIT(PIPE_B),
931 .dbuf_mask = {
932 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
933 },
934 },
935 {
936 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
937 .dbuf_mask = {
938 [PIPE_A] = BIT(DBUF_S1),
939 [PIPE_B] = BIT(DBUF_S2),
940 },
941 },
942 {
943 .active_pipes = BIT(PIPE_C),
944 .dbuf_mask = {
945 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
946 },
947 },
948 {
949 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
950 .dbuf_mask = {
951 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
952 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
953 },
954 },
955 {
956 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
957 .dbuf_mask = {
958 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
959 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
960 },
961 },
962 {
963 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
964 .dbuf_mask = {
965 [PIPE_A] = BIT(DBUF_S1),
966 [PIPE_B] = BIT(DBUF_S2),
967 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
968 },
969 },
970 {
971 .active_pipes = BIT(PIPE_D),
972 .dbuf_mask = {
973 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
974 },
975 },
976 {
977 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
978 .dbuf_mask = {
979 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
980 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
981 },
982 },
983 {
984 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
985 .dbuf_mask = {
986 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
987 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
988 },
989 },
990 {
991 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
992 .dbuf_mask = {
993 [PIPE_A] = BIT(DBUF_S1),
994 [PIPE_B] = BIT(DBUF_S2),
995 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
996 },
997 },
998 {
999 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1000 .dbuf_mask = {
1001 [PIPE_C] = BIT(DBUF_S3),
1002 [PIPE_D] = BIT(DBUF_S4),
1003 },
1004 },
1005 {
1006 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1007 .dbuf_mask = {
1008 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1009 [PIPE_C] = BIT(DBUF_S3),
1010 [PIPE_D] = BIT(DBUF_S4),
1011 },
1012 },
1013 {
1014 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1015 .dbuf_mask = {
1016 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1017 [PIPE_C] = BIT(DBUF_S3),
1018 [PIPE_D] = BIT(DBUF_S4),
1019 },
1020 },
1021 {
1022 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1023 .dbuf_mask = {
1024 [PIPE_A] = BIT(DBUF_S1),
1025 [PIPE_B] = BIT(DBUF_S2),
1026 [PIPE_C] = BIT(DBUF_S3),
1027 [PIPE_D] = BIT(DBUF_S4),
1028 },
1029 },
1030 {}
1031};
1032
1033static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1034 /*
1035 * Keep the join_mbus cases first so check_mbus_joined()
1036 * will prefer them over the !join_mbus cases.
1037 */
1038 {
1039 .active_pipes = BIT(PIPE_A),
1040 .dbuf_mask = {
1041 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1042 },
1043 .join_mbus = true,
1044 },
1045 {
1046 .active_pipes = BIT(PIPE_B),
1047 .dbuf_mask = {
1048 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1049 },
1050 .join_mbus = true,
1051 },
1052 {
1053 .active_pipes = BIT(PIPE_A),
1054 .dbuf_mask = {
1055 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1056 },
1057 .join_mbus = false,
1058 },
1059 {
1060 .active_pipes = BIT(PIPE_B),
1061 .dbuf_mask = {
1062 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1063 },
1064 .join_mbus = false,
1065 },
1066 {
1067 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1068 .dbuf_mask = {
1069 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1070 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1071 },
1072 },
1073 {
1074 .active_pipes = BIT(PIPE_C),
1075 .dbuf_mask = {
1076 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1077 },
1078 },
1079 {
1080 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1081 .dbuf_mask = {
1082 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1083 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1084 },
1085 },
1086 {
1087 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1088 .dbuf_mask = {
1089 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1090 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1091 },
1092 },
1093 {
1094 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1095 .dbuf_mask = {
1096 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1097 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1098 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1099 },
1100 },
1101 {
1102 .active_pipes = BIT(PIPE_D),
1103 .dbuf_mask = {
1104 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1105 },
1106 },
1107 {
1108 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1109 .dbuf_mask = {
1110 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1111 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1112 },
1113 },
1114 {
1115 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1116 .dbuf_mask = {
1117 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1118 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1119 },
1120 },
1121 {
1122 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1123 .dbuf_mask = {
1124 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1125 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1126 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1127 },
1128 },
1129 {
1130 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1131 .dbuf_mask = {
1132 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1133 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1134 },
1135 },
1136 {
1137 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1138 .dbuf_mask = {
1139 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1140 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1141 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1142 },
1143 },
1144 {
1145 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1146 .dbuf_mask = {
1147 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1148 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1149 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1150 },
1151 },
1152 {
1153 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1154 .dbuf_mask = {
1155 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1156 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1157 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1158 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1159 },
1160 },
1161 {}
1162
1163};
1164
1165static bool check_mbus_joined(u8 active_pipes,
1166 const struct dbuf_slice_conf_entry *dbuf_slices)
1167{
1168 int i;
1169
1170 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1171 if (dbuf_slices[i].active_pipes == active_pipes)
1172 return dbuf_slices[i].join_mbus;
1173 }
1174 return false;
1175}
1176
1177static bool adlp_check_mbus_joined(u8 active_pipes)
1178{
1179 return check_mbus_joined(active_pipes, dbuf_slices: adlp_allowed_dbufs);
1180}
1181
1182static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1183 const struct dbuf_slice_conf_entry *dbuf_slices)
1184{
1185 int i;
1186
1187 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1188 if (dbuf_slices[i].active_pipes == active_pipes &&
1189 dbuf_slices[i].join_mbus == join_mbus)
1190 return dbuf_slices[i].dbuf_mask[pipe];
1191 }
1192 return 0;
1193}
1194
1195/*
1196 * This function finds an entry with same enabled pipe configuration and
1197 * returns correspondent DBuf slice mask as stated in BSpec for particular
1198 * platform.
1199 */
1200static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1201{
1202 /*
1203 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1204 * required calculating "pipe ratio" in order to determine
1205 * if one or two slices can be used for single pipe configurations
1206 * as additional constraint to the existing table.
1207 * However based on recent info, it should be not "pipe ratio"
1208 * but rather ratio between pixel_rate and cdclk with additional
1209 * constants, so for now we are using only table until this is
1210 * clarified. Also this is the reason why crtc_state param is
1211 * still here - we will need it once those additional constraints
1212 * pop up.
1213 */
1214 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1215 dbuf_slices: icl_allowed_dbufs);
1216}
1217
1218static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1219{
1220 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1221 dbuf_slices: tgl_allowed_dbufs);
1222}
1223
1224static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1225{
1226 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1227 dbuf_slices: adlp_allowed_dbufs);
1228}
1229
1230static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1231{
1232 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1233 dbuf_slices: dg2_allowed_dbufs);
1234}
1235
1236static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1237{
1238 struct intel_display *display = to_intel_display(crtc);
1239 enum pipe pipe = crtc->pipe;
1240
1241 if (display->platform.dg2)
1242 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1243 else if (DISPLAY_VER(display) >= 13)
1244 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1245 else if (DISPLAY_VER(display) == 12)
1246 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1247 else if (DISPLAY_VER(display) == 11)
1248 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1249 /*
1250 * For anything else just return one slice yet.
1251 * Should be extended for other platforms.
1252 */
1253 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1254}
1255
1256static bool
1257use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1258 struct intel_plane *plane)
1259{
1260 struct intel_display *display = to_intel_display(plane);
1261
1262 /* Xe3+ are auto minimum DDB capble. So don't force minimal wm0 */
1263 return IS_DISPLAY_VER(display, 13, 20) &&
1264 crtc_state->uapi.async_flip &&
1265 plane->async_flip;
1266}
1267
1268unsigned int
1269skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
1270 struct intel_plane *plane, int width, int height,
1271 int cpp)
1272{
1273 /*
1274 * We calculate extra ddb based on ratio plane rate/total data rate
1275 * in case, in some cases we should not allocate extra ddb for the plane,
1276 * so do not count its data rate, if this is the case.
1277 */
1278 if (use_minimal_wm0_only(crtc_state, plane))
1279 return 0;
1280
1281 return width * height * cpp;
1282}
1283
1284static u64
1285skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1286{
1287 struct intel_display *display = to_intel_display(crtc_state);
1288 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1289 enum plane_id plane_id;
1290 u64 data_rate = 0;
1291
1292 for_each_plane_id_on_crtc(crtc, plane_id) {
1293 if (plane_id == PLANE_CURSOR)
1294 continue;
1295
1296 data_rate += crtc_state->rel_data_rate[plane_id];
1297
1298 if (DISPLAY_VER(display) < 11)
1299 data_rate += crtc_state->rel_data_rate_y[plane_id];
1300 }
1301
1302 return data_rate;
1303}
1304
1305const struct skl_wm_level *
1306skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1307 enum plane_id plane_id,
1308 int level)
1309{
1310 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1311
1312 if (level == 0 && pipe_wm->use_sagv_wm)
1313 return &wm->sagv.wm0;
1314
1315 return &wm->wm[level];
1316}
1317
1318const struct skl_wm_level *
1319skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1320 enum plane_id plane_id)
1321{
1322 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1323
1324 if (pipe_wm->use_sagv_wm)
1325 return &wm->sagv.trans_wm;
1326
1327 return &wm->trans_wm;
1328}
1329
1330/*
1331 * We only disable the watermarks for each plane if
1332 * they exceed the ddb allocation of said plane. This
1333 * is done so that we don't end up touching cursor
1334 * watermarks needlessly when some other plane reduces
1335 * our max possible watermark level.
1336 *
1337 * Bspec has this to say about the PLANE_WM enable bit:
1338 * "All the watermarks at this level for all enabled
1339 * planes must be enabled before the level will be used."
1340 * So this is actually safe to do.
1341 */
1342static void
1343skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1344{
1345 if (wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb))
1346 memset(s: wm, c: 0, n: sizeof(*wm));
1347}
1348
1349static void
1350skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1351 const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1352{
1353 if (wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb_y) ||
1354 uv_wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb)) {
1355 memset(s: wm, c: 0, n: sizeof(*wm));
1356 memset(s: uv_wm, c: 0, n: sizeof(*uv_wm));
1357 }
1358}
1359
1360static bool skl_need_wm_copy_wa(struct intel_display *display, int level,
1361 const struct skl_plane_wm *wm)
1362{
1363 /*
1364 * Wa_1408961008:icl, ehl
1365 * Wa_14012656716:tgl, adl
1366 * Wa_14017887344:icl
1367 * Wa_14017868169:adl, tgl
1368 * Due to some power saving optimizations, different subsystems
1369 * like PSR, might still use even disabled wm level registers,
1370 * for "reference", so lets keep at least the values sane.
1371 * Considering amount of WA requiring us to do similar things, was
1372 * decided to simply do it for all of the platforms, as those wm
1373 * levels are disabled, this isn't going to do harm anyway.
1374 */
1375 return level > 0 && !wm->wm[level].enable;
1376}
1377
1378struct skl_plane_ddb_iter {
1379 u64 data_rate;
1380 u16 start, size;
1381};
1382
1383static void
1384skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1385 struct skl_ddb_entry *ddb,
1386 const struct skl_wm_level *wm,
1387 u64 data_rate)
1388{
1389 u16 size, extra = 0;
1390
1391 if (data_rate && iter->data_rate) {
1392 extra = min_t(u16, iter->size,
1393 DIV64_U64_ROUND_UP(iter->size * data_rate,
1394 iter->data_rate));
1395 iter->size -= extra;
1396 iter->data_rate -= data_rate;
1397 }
1398
1399 /*
1400 * Keep ddb entry of all disabled planes explicitly zeroed
1401 * to avoid skl_ddb_add_affected_planes() adding them to
1402 * the state when other planes change their allocations.
1403 */
1404 size = wm->min_ddb_alloc + extra;
1405 if (size)
1406 iter->start = skl_ddb_entry_init(entry: ddb, start: iter->start,
1407 end: iter->start + size);
1408}
1409
1410static int
1411skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1412 struct intel_crtc *crtc)
1413{
1414 struct intel_crtc_state *crtc_state =
1415 intel_atomic_get_new_crtc_state(state, crtc);
1416 const struct intel_dbuf_state *dbuf_state =
1417 intel_atomic_get_new_dbuf_state(state);
1418 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1419 struct intel_display *display = to_intel_display(state);
1420 int num_active = hweight8(dbuf_state->active_pipes);
1421 struct skl_plane_ddb_iter iter;
1422 enum plane_id plane_id;
1423 u16 cursor_size;
1424 u32 blocks;
1425 int level;
1426
1427 /* Clear the partitioning for disabled planes. */
1428 memset(s: crtc_state->wm.skl.plane_ddb, c: 0, n: sizeof(crtc_state->wm.skl.plane_ddb));
1429 memset(s: crtc_state->wm.skl.plane_ddb_y, c: 0, n: sizeof(crtc_state->wm.skl.plane_ddb_y));
1430 memset(s: crtc_state->wm.skl.plane_min_ddb, c: 0,
1431 n: sizeof(crtc_state->wm.skl.plane_min_ddb));
1432 memset(s: crtc_state->wm.skl.plane_interim_ddb, c: 0,
1433 n: sizeof(crtc_state->wm.skl.plane_interim_ddb));
1434
1435 if (!crtc_state->hw.active)
1436 return 0;
1437
1438 iter.start = alloc->start;
1439 iter.size = skl_ddb_entry_size(entry: alloc);
1440 if (iter.size == 0)
1441 return 0;
1442
1443 /* Allocate fixed number of blocks for cursor. */
1444 cursor_size = skl_cursor_allocation(crtc_state, num_active);
1445 iter.size -= cursor_size;
1446 skl_ddb_entry_init(entry: &crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1447 start: alloc->end - cursor_size, end: alloc->end);
1448
1449 iter.data_rate = skl_total_relative_data_rate(crtc_state);
1450
1451 /*
1452 * Find the highest watermark level for which we can satisfy the block
1453 * requirement of active planes.
1454 */
1455 for (level = display->wm.num_levels - 1; level >= 0; level--) {
1456 blocks = 0;
1457 for_each_plane_id_on_crtc(crtc, plane_id) {
1458 const struct skl_plane_wm *wm =
1459 &crtc_state->wm.skl.optimal.planes[plane_id];
1460
1461 if (plane_id == PLANE_CURSOR) {
1462 const struct skl_ddb_entry *ddb =
1463 &crtc_state->wm.skl.plane_ddb[plane_id];
1464
1465 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(entry: ddb)) {
1466 drm_WARN_ON(display->drm,
1467 wm->wm[level].min_ddb_alloc != U16_MAX);
1468 blocks = U32_MAX;
1469 break;
1470 }
1471 continue;
1472 }
1473
1474 blocks += wm->wm[level].min_ddb_alloc;
1475 blocks += wm->uv_wm[level].min_ddb_alloc;
1476 }
1477
1478 if (blocks <= iter.size) {
1479 iter.size -= blocks;
1480 break;
1481 }
1482 }
1483
1484 if (level < 0) {
1485 drm_dbg_kms(display->drm,
1486 "Requested display configuration exceeds system DDB limitations");
1487 drm_dbg_kms(display->drm, "minimum required %d/%d\n",
1488 blocks, iter.size);
1489 return -EINVAL;
1490 }
1491
1492 /* avoid the WARN later when we don't allocate any extra DDB */
1493 if (iter.data_rate == 0)
1494 iter.size = 0;
1495
1496 /*
1497 * Grant each plane the blocks it requires at the highest achievable
1498 * watermark level, plus an extra share of the leftover blocks
1499 * proportional to its relative data rate.
1500 */
1501 for_each_plane_id_on_crtc(crtc, plane_id) {
1502 struct skl_ddb_entry *ddb =
1503 &crtc_state->wm.skl.plane_ddb[plane_id];
1504 struct skl_ddb_entry *ddb_y =
1505 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1506 u16 *min_ddb = &crtc_state->wm.skl.plane_min_ddb[plane_id];
1507 u16 *interim_ddb =
1508 &crtc_state->wm.skl.plane_interim_ddb[plane_id];
1509 const struct skl_plane_wm *wm =
1510 &crtc_state->wm.skl.optimal.planes[plane_id];
1511
1512 if (plane_id == PLANE_CURSOR)
1513 continue;
1514
1515 if (DISPLAY_VER(display) < 11 &&
1516 crtc_state->nv12_planes & BIT(plane_id)) {
1517 skl_allocate_plane_ddb(iter: &iter, ddb: ddb_y, wm: &wm->wm[level],
1518 data_rate: crtc_state->rel_data_rate_y[plane_id]);
1519 skl_allocate_plane_ddb(iter: &iter, ddb, wm: &wm->uv_wm[level],
1520 data_rate: crtc_state->rel_data_rate[plane_id]);
1521 } else {
1522 skl_allocate_plane_ddb(iter: &iter, ddb, wm: &wm->wm[level],
1523 data_rate: crtc_state->rel_data_rate[plane_id]);
1524 }
1525
1526 if (DISPLAY_VER(display) >= 30) {
1527 *min_ddb = wm->wm[0].min_ddb_alloc;
1528 *interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1529 }
1530 }
1531 drm_WARN_ON(display->drm, iter.size != 0 || iter.data_rate != 0);
1532
1533 /*
1534 * When we calculated watermark values we didn't know how high
1535 * of a level we'd actually be able to hit, so we just marked
1536 * all levels as "enabled." Go back now and disable the ones
1537 * that aren't actually possible.
1538 */
1539 for (level++; level < display->wm.num_levels; level++) {
1540 for_each_plane_id_on_crtc(crtc, plane_id) {
1541 const struct skl_ddb_entry *ddb =
1542 &crtc_state->wm.skl.plane_ddb[plane_id];
1543 const struct skl_ddb_entry *ddb_y =
1544 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1545 struct skl_plane_wm *wm =
1546 &crtc_state->wm.skl.optimal.planes[plane_id];
1547
1548 if (DISPLAY_VER(display) < 11 &&
1549 crtc_state->nv12_planes & BIT(plane_id))
1550 skl_check_nv12_wm_level(wm: &wm->wm[level],
1551 uv_wm: &wm->uv_wm[level],
1552 ddb_y, ddb);
1553 else
1554 skl_check_wm_level(wm: &wm->wm[level], ddb);
1555
1556 if (skl_need_wm_copy_wa(display, level, wm)) {
1557 wm->wm[level].blocks = wm->wm[level - 1].blocks;
1558 wm->wm[level].lines = wm->wm[level - 1].lines;
1559 wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1560 }
1561 }
1562 }
1563
1564 /*
1565 * Go back and disable the transition and SAGV watermarks
1566 * if it turns out we don't have enough DDB blocks for them.
1567 */
1568 for_each_plane_id_on_crtc(crtc, plane_id) {
1569 const struct skl_ddb_entry *ddb =
1570 &crtc_state->wm.skl.plane_ddb[plane_id];
1571 const struct skl_ddb_entry *ddb_y =
1572 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1573 u16 *interim_ddb =
1574 &crtc_state->wm.skl.plane_interim_ddb[plane_id];
1575 struct skl_plane_wm *wm =
1576 &crtc_state->wm.skl.optimal.planes[plane_id];
1577
1578 if (DISPLAY_VER(display) < 11 &&
1579 crtc_state->nv12_planes & BIT(plane_id)) {
1580 skl_check_wm_level(wm: &wm->trans_wm, ddb: ddb_y);
1581 } else {
1582 WARN_ON(skl_ddb_entry_size(ddb_y));
1583
1584 skl_check_wm_level(wm: &wm->trans_wm, ddb);
1585 }
1586
1587 skl_check_wm_level(wm: &wm->sagv.wm0, ddb);
1588 if (DISPLAY_VER(display) >= 30)
1589 *interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1590
1591 skl_check_wm_level(wm: &wm->sagv.trans_wm, ddb);
1592 }
1593
1594 return 0;
1595}
1596
1597/*
1598 * The max latency should be 257 (max the punit can code is 255 and we add 2us
1599 * for the read latency) and cpp should always be <= 8, so that
1600 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1601 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1602 */
1603static uint_fixed_16_16_t
1604skl_wm_method1(struct intel_display *display, u32 pixel_rate,
1605 u8 cpp, u32 latency, u32 dbuf_block_size)
1606{
1607 u32 wm_intermediate_val;
1608 uint_fixed_16_16_t ret;
1609
1610 if (latency == 0)
1611 return FP_16_16_MAX;
1612
1613 wm_intermediate_val = latency * pixel_rate * cpp;
1614 ret = div_fixed16(val: wm_intermediate_val, d: 1000 * dbuf_block_size);
1615
1616 if (DISPLAY_VER(display) >= 10)
1617 ret = add_fixed16_u32(add1: ret, add2: 1);
1618
1619 return ret;
1620}
1621
1622static uint_fixed_16_16_t
1623skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1624 uint_fixed_16_16_t plane_blocks_per_line)
1625{
1626 u32 wm_intermediate_val;
1627 uint_fixed_16_16_t ret;
1628
1629 if (latency == 0)
1630 return FP_16_16_MAX;
1631
1632 wm_intermediate_val = latency * pixel_rate;
1633 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1634 pipe_htotal * 1000);
1635 ret = mul_u32_fixed16(val: wm_intermediate_val, mul: plane_blocks_per_line);
1636 return ret;
1637}
1638
1639static uint_fixed_16_16_t
1640intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1641{
1642 struct intel_display *display = to_intel_display(crtc_state);
1643 u32 pixel_rate;
1644 u32 crtc_htotal;
1645 uint_fixed_16_16_t linetime_us;
1646
1647 if (!crtc_state->hw.active)
1648 return u32_to_fixed16(val: 0);
1649
1650 pixel_rate = crtc_state->pixel_rate;
1651
1652 if (drm_WARN_ON(display->drm, pixel_rate == 0))
1653 return u32_to_fixed16(val: 0);
1654
1655 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1656 linetime_us = div_fixed16(val: crtc_htotal * 1000, d: pixel_rate);
1657
1658 return linetime_us;
1659}
1660
1661static int
1662skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1663 int width, const struct drm_format_info *format,
1664 u64 modifier, unsigned int rotation,
1665 u32 plane_pixel_rate, struct skl_wm_params *wp,
1666 int color_plane, unsigned int pan_x)
1667{
1668 struct intel_display *display = to_intel_display(crtc_state);
1669 u32 interm_pbpl;
1670
1671 /* only planar format has two planes */
1672 if (color_plane == 1 &&
1673 !intel_format_info_is_yuv_semiplanar(info: format, modifier)) {
1674 drm_dbg_kms(display->drm,
1675 "Non planar format have single plane\n");
1676 return -EINVAL;
1677 }
1678
1679 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1680 wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1681 intel_fb_is_tiled_modifier(modifier);
1682 wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1683 wp->is_planar = intel_format_info_is_yuv_semiplanar(info: format, modifier);
1684
1685 wp->width = width;
1686 if (color_plane == 1 && wp->is_planar)
1687 wp->width /= 2;
1688
1689 wp->cpp = format->cpp[color_plane];
1690 wp->plane_pixel_rate = plane_pixel_rate;
1691
1692 if (DISPLAY_VER(display) >= 11 &&
1693 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
1694 wp->dbuf_block_size = 256;
1695 else
1696 wp->dbuf_block_size = 512;
1697
1698 if (drm_rotation_90_or_270(rotation)) {
1699 switch (wp->cpp) {
1700 case 1:
1701 wp->y_min_scanlines = 16;
1702 break;
1703 case 2:
1704 wp->y_min_scanlines = 8;
1705 break;
1706 case 4:
1707 wp->y_min_scanlines = 4;
1708 break;
1709 default:
1710 MISSING_CASE(wp->cpp);
1711 return -EINVAL;
1712 }
1713 } else {
1714 wp->y_min_scanlines = 4;
1715 }
1716
1717 if (skl_needs_memory_bw_wa(display))
1718 wp->y_min_scanlines *= 2;
1719
1720 wp->plane_bytes_per_line = wp->width * wp->cpp;
1721 if (wp->y_tiled) {
1722 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1723 wp->y_min_scanlines,
1724 wp->dbuf_block_size);
1725
1726 if (DISPLAY_VER(display) >= 30)
1727 interm_pbpl += (pan_x != 0);
1728 else if (DISPLAY_VER(display) >= 10)
1729 interm_pbpl++;
1730
1731 wp->plane_blocks_per_line = div_fixed16(val: interm_pbpl,
1732 d: wp->y_min_scanlines);
1733 } else {
1734 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1735 wp->dbuf_block_size);
1736
1737 if (!wp->x_tiled || DISPLAY_VER(display) >= 10)
1738 interm_pbpl++;
1739
1740 wp->plane_blocks_per_line = u32_to_fixed16(val: interm_pbpl);
1741 }
1742
1743 wp->y_tile_minimum = mul_u32_fixed16(val: wp->y_min_scanlines,
1744 mul: wp->plane_blocks_per_line);
1745
1746 wp->linetime_us = fixed16_to_u32_round_up(fp: intel_get_linetime_us(crtc_state));
1747
1748 return 0;
1749}
1750
1751static int
1752skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1753 const struct intel_plane_state *plane_state,
1754 struct skl_wm_params *wp, int color_plane)
1755{
1756 const struct drm_framebuffer *fb = plane_state->hw.fb;
1757 int width;
1758
1759 /*
1760 * Src coordinates are already rotated by 270 degrees for
1761 * the 90/270 degree plane rotation cases (to match the
1762 * GTT mapping), hence no need to account for rotation here.
1763 */
1764 width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1765
1766 return skl_compute_wm_params(crtc_state, width,
1767 format: fb->format, modifier: fb->modifier,
1768 rotation: plane_state->hw.rotation,
1769 plane_pixel_rate: intel_plane_pixel_rate(crtc_state, plane_state),
1770 wp, color_plane,
1771 pan_x: plane_state->uapi.src.x1);
1772}
1773
1774static bool skl_wm_has_lines(struct intel_display *display, int level)
1775{
1776 if (DISPLAY_VER(display) >= 10)
1777 return true;
1778
1779 /* The number of lines are ignored for the level 0 watermark. */
1780 return level > 0;
1781}
1782
1783static int skl_wm_max_lines(struct intel_display *display)
1784{
1785 if (DISPLAY_VER(display) >= 13)
1786 return 255;
1787 else
1788 return 31;
1789}
1790
1791static bool xe3_auto_min_alloc_capable(struct intel_plane *plane, int level)
1792{
1793 struct intel_display *display = to_intel_display(plane);
1794
1795 return DISPLAY_VER(display) >= 30 && level == 0 && plane->id != PLANE_CURSOR;
1796}
1797
1798static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1799 struct intel_plane *plane,
1800 int level,
1801 unsigned int latency,
1802 const struct skl_wm_params *wp,
1803 const struct skl_wm_level *result_prev,
1804 struct skl_wm_level *result /* out */)
1805{
1806 struct intel_display *display = to_intel_display(crtc_state);
1807 uint_fixed_16_16_t method1, method2;
1808 uint_fixed_16_16_t selected_result;
1809 u32 blocks, lines, min_ddb_alloc = 0;
1810
1811 if (latency == 0 ||
1812 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1813 /* reject it */
1814 result->min_ddb_alloc = U16_MAX;
1815 return;
1816 }
1817
1818 method1 = skl_wm_method1(display, pixel_rate: wp->plane_pixel_rate,
1819 cpp: wp->cpp, latency, dbuf_block_size: wp->dbuf_block_size);
1820 method2 = skl_wm_method2(pixel_rate: wp->plane_pixel_rate,
1821 pipe_htotal: crtc_state->hw.pipe_mode.crtc_htotal,
1822 latency,
1823 plane_blocks_per_line: wp->plane_blocks_per_line);
1824
1825 if (wp->y_tiled) {
1826 selected_result = max_fixed16(max1: method2, max2: wp->y_tile_minimum);
1827 } else {
1828 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1829 wp->dbuf_block_size < 1) &&
1830 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1831 selected_result = method2;
1832 } else if (latency >= wp->linetime_us) {
1833 if (DISPLAY_VER(display) == 9)
1834 selected_result = min_fixed16(min1: method1, min2: method2);
1835 else
1836 selected_result = method2;
1837 } else {
1838 selected_result = method1;
1839 }
1840 }
1841
1842 blocks = fixed16_to_u32_round_up(fp: selected_result);
1843 if (DISPLAY_VER(display) < 30)
1844 blocks++;
1845
1846 /*
1847 * Lets have blocks at minimum equivalent to plane_blocks_per_line
1848 * as there will be at minimum one line for lines configuration. This
1849 * is a work around for FIFO underruns observed with resolutions like
1850 * 4k 60 Hz in single channel DRAM configurations.
1851 *
1852 * As per the Bspec 49325, if the ddb allocation can hold at least
1853 * one plane_blocks_per_line, we should have selected method2 in
1854 * the above logic. Assuming that modern versions have enough dbuf
1855 * and method2 guarantees blocks equivalent to at least 1 line,
1856 * select the blocks as plane_blocks_per_line.
1857 *
1858 * TODO: Revisit the logic when we have better understanding on DRAM
1859 * channels' impact on the level 0 memory latency and the relevant
1860 * wm calculations.
1861 */
1862 if (skl_wm_has_lines(display, level))
1863 blocks = max(blocks,
1864 fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1865 lines = div_round_up_fixed16(val: selected_result,
1866 d: wp->plane_blocks_per_line);
1867
1868 if (DISPLAY_VER(display) == 9) {
1869 /* Display WA #1125: skl,bxt,kbl */
1870 if (level == 0 && wp->rc_surface)
1871 blocks += fixed16_to_u32_round_up(fp: wp->y_tile_minimum);
1872
1873 /* Display WA #1126: skl,bxt,kbl */
1874 if (level >= 1 && level <= 7) {
1875 if (wp->y_tiled) {
1876 blocks += fixed16_to_u32_round_up(fp: wp->y_tile_minimum);
1877 lines += wp->y_min_scanlines;
1878 } else {
1879 blocks++;
1880 }
1881
1882 /*
1883 * Make sure result blocks for higher latency levels are
1884 * at least as high as level below the current level.
1885 * Assumption in DDB algorithm optimization for special
1886 * cases. Also covers Display WA #1125 for RC.
1887 */
1888 if (result_prev->blocks > blocks)
1889 blocks = result_prev->blocks;
1890 }
1891 }
1892
1893 if (DISPLAY_VER(display) >= 11) {
1894 if (wp->y_tiled) {
1895 int extra_lines;
1896
1897 if (lines % wp->y_min_scanlines == 0)
1898 extra_lines = wp->y_min_scanlines;
1899 else
1900 extra_lines = wp->y_min_scanlines * 2 -
1901 lines % wp->y_min_scanlines;
1902
1903 min_ddb_alloc = mul_round_up_u32_fixed16(val: lines + extra_lines,
1904 mul: wp->plane_blocks_per_line);
1905 } else {
1906 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1907 }
1908 }
1909
1910 if (!skl_wm_has_lines(display, level))
1911 lines = 0;
1912
1913 if (lines > skl_wm_max_lines(display)) {
1914 /* reject it */
1915 result->min_ddb_alloc = U16_MAX;
1916 return;
1917 }
1918
1919 /*
1920 * If lines is valid, assume we can use this watermark level
1921 * for now. We'll come back and disable it after we calculate the
1922 * DDB allocation if it turns out we don't actually have enough
1923 * blocks to satisfy it.
1924 */
1925 result->blocks = blocks;
1926 result->lines = lines;
1927 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1928 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1929 result->enable = true;
1930 result->auto_min_alloc_wm_enable = xe3_auto_min_alloc_capable(plane, level);
1931
1932 if (DISPLAY_VER(display) < 12 && display->sagv.block_time_us)
1933 result->can_sagv = latency >= display->sagv.block_time_us;
1934}
1935
1936static void
1937skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1938 struct intel_plane *plane,
1939 const struct skl_wm_params *wm_params,
1940 struct skl_wm_level *levels)
1941{
1942 struct intel_display *display = to_intel_display(crtc_state);
1943 struct skl_wm_level *result_prev = &levels[0];
1944 int level;
1945
1946 for (level = 0; level < display->wm.num_levels; level++) {
1947 struct skl_wm_level *result = &levels[level];
1948 unsigned int latency = skl_wm_latency(display, level, wp: wm_params);
1949
1950 skl_compute_plane_wm(crtc_state, plane, level, latency,
1951 wp: wm_params, result_prev, result);
1952
1953 result_prev = result;
1954 }
1955}
1956
1957static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
1958 struct intel_plane *plane,
1959 const struct skl_wm_params *wm_params,
1960 struct skl_plane_wm *plane_wm)
1961{
1962 struct intel_display *display = to_intel_display(crtc_state);
1963 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
1964 struct skl_wm_level *levels = plane_wm->wm;
1965 unsigned int latency = 0;
1966
1967 if (display->sagv.block_time_us)
1968 latency = display->sagv.block_time_us +
1969 skl_wm_latency(display, level: 0, wp: wm_params);
1970
1971 skl_compute_plane_wm(crtc_state, plane, level: 0, latency,
1972 wp: wm_params, result_prev: &levels[0],
1973 result: sagv_wm);
1974}
1975
1976static void skl_compute_transition_wm(struct intel_display *display,
1977 struct skl_wm_level *trans_wm,
1978 const struct skl_wm_level *wm0,
1979 const struct skl_wm_params *wp)
1980{
1981 u16 trans_min, trans_amount, trans_y_tile_min;
1982 u16 wm0_blocks, trans_offset, blocks;
1983
1984 /* Transition WM don't make any sense if ipc is disabled */
1985 if (!skl_watermark_ipc_enabled(display))
1986 return;
1987
1988 /*
1989 * WaDisableTWM:skl,kbl,cfl,bxt
1990 * Transition WM are not recommended by HW team for GEN9
1991 */
1992 if (DISPLAY_VER(display) == 9)
1993 return;
1994
1995 if (DISPLAY_VER(display) >= 11)
1996 trans_min = 4;
1997 else
1998 trans_min = 14;
1999
2000 /* Display WA #1140: glk,cnl */
2001 if (DISPLAY_VER(display) == 10)
2002 trans_amount = 0;
2003 else
2004 trans_amount = 10; /* This is configurable amount */
2005
2006 trans_offset = trans_min + trans_amount;
2007
2008 /*
2009 * The spec asks for Selected Result Blocks for wm0 (the real value),
2010 * not Result Blocks (the integer value). Pay attention to the capital
2011 * letters. The value wm_l0->blocks is actually Result Blocks, but
2012 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2013 * and since we later will have to get the ceiling of the sum in the
2014 * transition watermarks calculation, we can just pretend Selected
2015 * Result Blocks is Result Blocks minus 1 and it should work for the
2016 * current platforms.
2017 */
2018 wm0_blocks = wm0->blocks - 1;
2019
2020 if (wp->y_tiled) {
2021 trans_y_tile_min =
2022 (u16)mul_round_up_u32_fixed16(val: 2, mul: wp->y_tile_minimum);
2023 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2024 } else {
2025 blocks = wm0_blocks + trans_offset;
2026 }
2027 blocks++;
2028
2029 /*
2030 * Just assume we can enable the transition watermark. After
2031 * computing the DDB we'll come back and disable it if that
2032 * assumption turns out to be false.
2033 */
2034 trans_wm->blocks = blocks;
2035 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2036 trans_wm->enable = true;
2037}
2038
2039static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2040 const struct intel_plane_state *plane_state,
2041 struct intel_plane *plane, int color_plane)
2042{
2043 struct intel_display *display = to_intel_display(crtc_state);
2044 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2045 struct skl_wm_params wm_params;
2046 int ret;
2047
2048 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2049 wp: &wm_params, color_plane);
2050 if (ret)
2051 return ret;
2052
2053 skl_compute_wm_levels(crtc_state, plane, wm_params: &wm_params, levels: wm->wm);
2054
2055 skl_compute_transition_wm(display, trans_wm: &wm->trans_wm,
2056 wm0: &wm->wm[0], wp: &wm_params);
2057
2058 if (DISPLAY_VER(display) >= 12) {
2059 tgl_compute_sagv_wm(crtc_state, plane, wm_params: &wm_params, plane_wm: wm);
2060
2061 skl_compute_transition_wm(display, trans_wm: &wm->sagv.trans_wm,
2062 wm0: &wm->sagv.wm0, wp: &wm_params);
2063 }
2064
2065 return 0;
2066}
2067
2068static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2069 const struct intel_plane_state *plane_state,
2070 struct intel_plane *plane)
2071{
2072 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2073 struct skl_wm_params wm_params;
2074 int ret;
2075
2076 wm->is_planar = true;
2077
2078 /* uv plane watermarks must also be validated for NV12/Planar */
2079 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2080 wp: &wm_params, color_plane: 1);
2081 if (ret)
2082 return ret;
2083
2084 skl_compute_wm_levels(crtc_state, plane, wm_params: &wm_params, levels: wm->uv_wm);
2085
2086 return 0;
2087}
2088
2089static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2090 const struct intel_plane_state *plane_state)
2091{
2092 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2093 enum plane_id plane_id = plane->id;
2094 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2095 const struct drm_framebuffer *fb = plane_state->hw.fb;
2096 int ret;
2097
2098 memset(s: wm, c: 0, n: sizeof(*wm));
2099
2100 if (!intel_wm_plane_visible(crtc_state, plane_state))
2101 return 0;
2102
2103 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2104 plane, color_plane: 0);
2105 if (ret)
2106 return ret;
2107
2108 if (fb->format->is_yuv && fb->format->num_planes > 1) {
2109 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2110 plane);
2111 if (ret)
2112 return ret;
2113 }
2114
2115 return 0;
2116}
2117
2118static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2119 const struct intel_plane_state *plane_state)
2120{
2121 struct intel_display *display = to_intel_display(plane_state);
2122 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2123 enum plane_id plane_id = plane->id;
2124 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2125 int ret;
2126
2127 /* Watermarks calculated on UV plane */
2128 if (plane_state->is_y_plane)
2129 return 0;
2130
2131 memset(s: wm, c: 0, n: sizeof(*wm));
2132
2133 if (plane_state->planar_linked_plane) {
2134 const struct drm_framebuffer *fb = plane_state->hw.fb;
2135
2136 drm_WARN_ON(display->drm,
2137 !intel_wm_plane_visible(crtc_state, plane_state));
2138 drm_WARN_ON(display->drm, !fb->format->is_yuv ||
2139 fb->format->num_planes == 1);
2140
2141 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2142 plane: plane_state->planar_linked_plane, color_plane: 0);
2143 if (ret)
2144 return ret;
2145
2146 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2147 plane, color_plane: 1);
2148 if (ret)
2149 return ret;
2150 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2151 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2152 plane, color_plane: 0);
2153 if (ret)
2154 return ret;
2155 }
2156
2157 return 0;
2158}
2159
2160static int
2161cdclk_prefill_adjustment(const struct intel_crtc_state *crtc_state)
2162{
2163 struct intel_display *display = to_intel_display(crtc_state);
2164 struct intel_atomic_state *state =
2165 to_intel_atomic_state(crtc_state->uapi.state);
2166 const struct intel_cdclk_state *cdclk_state;
2167
2168 cdclk_state = intel_atomic_get_cdclk_state(state);
2169 if (IS_ERR(ptr: cdclk_state)) {
2170 drm_WARN_ON(display->drm, PTR_ERR(cdclk_state));
2171 return 1;
2172 }
2173
2174 return min(1, DIV_ROUND_UP(crtc_state->pixel_rate,
2175 2 * intel_cdclk_logical(cdclk_state)));
2176}
2177
2178static int
2179dsc_prefill_latency(const struct intel_crtc_state *crtc_state)
2180{
2181 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2182 const struct intel_crtc_scaler_state *scaler_state =
2183 &crtc_state->scaler_state;
2184 int linetime = DIV_ROUND_UP(1000 * crtc_state->hw.adjusted_mode.htotal,
2185 crtc_state->hw.adjusted_mode.clock);
2186 int num_scaler_users = hweight32(scaler_state->scaler_users);
2187 int chroma_downscaling_factor =
2188 crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ? 2 : 1;
2189 u32 dsc_prefill_latency = 0;
2190
2191 if (!crtc_state->dsc.compression_enable ||
2192 !num_scaler_users ||
2193 num_scaler_users > crtc->num_scalers)
2194 return dsc_prefill_latency;
2195
2196 dsc_prefill_latency = DIV_ROUND_UP(15 * linetime * chroma_downscaling_factor, 10);
2197
2198 for (int i = 0; i < num_scaler_users; i++) {
2199 u64 hscale_k, vscale_k;
2200
2201 hscale_k = max(1000, mul_u32_u32(scaler_state->scalers[i].hscale, 1000) >> 16);
2202 vscale_k = max(1000, mul_u32_u32(scaler_state->scalers[i].vscale, 1000) >> 16);
2203 dsc_prefill_latency = DIV_ROUND_UP_ULL(dsc_prefill_latency * hscale_k * vscale_k,
2204 1000000);
2205 }
2206
2207 dsc_prefill_latency *= cdclk_prefill_adjustment(crtc_state);
2208
2209 return intel_usecs_to_scanlines(adjusted_mode: &crtc_state->hw.adjusted_mode, usecs: dsc_prefill_latency);
2210}
2211
2212static int
2213scaler_prefill_latency(const struct intel_crtc_state *crtc_state)
2214{
2215 const struct intel_crtc_scaler_state *scaler_state =
2216 &crtc_state->scaler_state;
2217 int num_scaler_users = hweight32(scaler_state->scaler_users);
2218 int scaler_prefill_latency = 0;
2219 int linetime = DIV_ROUND_UP(1000 * crtc_state->hw.adjusted_mode.htotal,
2220 crtc_state->hw.adjusted_mode.clock);
2221
2222 if (!num_scaler_users)
2223 return scaler_prefill_latency;
2224
2225 scaler_prefill_latency = 4 * linetime;
2226
2227 if (num_scaler_users > 1) {
2228 u64 hscale_k = max(1000, mul_u32_u32(scaler_state->scalers[0].hscale, 1000) >> 16);
2229 u64 vscale_k = max(1000, mul_u32_u32(scaler_state->scalers[0].vscale, 1000) >> 16);
2230 int chroma_downscaling_factor =
2231 crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ? 2 : 1;
2232 int latency;
2233
2234 latency = DIV_ROUND_UP_ULL((4 * linetime * hscale_k * vscale_k *
2235 chroma_downscaling_factor), 1000000);
2236 scaler_prefill_latency += latency;
2237 }
2238
2239 scaler_prefill_latency *= cdclk_prefill_adjustment(crtc_state);
2240
2241 return intel_usecs_to_scanlines(adjusted_mode: &crtc_state->hw.adjusted_mode, usecs: scaler_prefill_latency);
2242}
2243
2244static bool
2245skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2246 int wm0_lines, int latency)
2247{
2248 const struct drm_display_mode *adjusted_mode =
2249 &crtc_state->hw.adjusted_mode;
2250
2251 return crtc_state->framestart_delay +
2252 intel_usecs_to_scanlines(adjusted_mode, usecs: latency) +
2253 scaler_prefill_latency(crtc_state) +
2254 dsc_prefill_latency(crtc_state) +
2255 wm0_lines >
2256 adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2257}
2258
2259static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2260{
2261 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2262 enum plane_id plane_id;
2263 int wm0_lines = 0;
2264
2265 for_each_plane_id_on_crtc(crtc, plane_id) {
2266 const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2267
2268 /* FIXME what about !skl_wm_has_lines() platforms? */
2269 wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2270 }
2271
2272 return wm0_lines;
2273}
2274
2275/*
2276 * TODO: In case we use PKG_C_LATENCY to allow C-states when the delayed vblank
2277 * size is too small for the package C exit latency we need to notify PSR about
2278 * the scenario to apply Wa_16025596647.
2279 */
2280static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2281 int wm0_lines)
2282{
2283 struct intel_display *display = to_intel_display(crtc_state);
2284 int level;
2285
2286 for (level = display->wm.num_levels - 1; level >= 0; level--) {
2287 int latency;
2288
2289 /* FIXME should we care about the latency w/a's? */
2290 latency = skl_wm_latency(display, level, NULL);
2291 if (latency == 0)
2292 continue;
2293
2294 /* FIXME is it correct to use 0 latency for wm0 here? */
2295 if (level == 0)
2296 latency = 0;
2297
2298 if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2299 return level;
2300 }
2301
2302 return -EINVAL;
2303}
2304
2305static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2306{
2307 struct intel_display *display = to_intel_display(crtc_state);
2308 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2309 int wm0_lines, level;
2310
2311 if (!crtc_state->hw.active)
2312 return 0;
2313
2314 wm0_lines = skl_max_wm0_lines(crtc_state);
2315
2316 level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2317 if (level < 0)
2318 return level;
2319
2320 /*
2321 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2322 * based on whether we're limited by the vblank duration.
2323 */
2324 crtc_state->wm_level_disabled = level < display->wm.num_levels - 1;
2325
2326 for (level++; level < display->wm.num_levels; level++) {
2327 enum plane_id plane_id;
2328
2329 for_each_plane_id_on_crtc(crtc, plane_id) {
2330 struct skl_plane_wm *wm =
2331 &crtc_state->wm.skl.optimal.planes[plane_id];
2332
2333 /*
2334 * FIXME just clear enable or flag the entire
2335 * thing as bad via min_ddb_alloc=U16_MAX?
2336 */
2337 wm->wm[level].enable = false;
2338 wm->uv_wm[level].enable = false;
2339 }
2340 }
2341
2342 if (DISPLAY_VER(display) >= 12 &&
2343 display->sagv.block_time_us &&
2344 skl_is_vblank_too_short(crtc_state, wm0_lines,
2345 latency: display->sagv.block_time_us)) {
2346 enum plane_id plane_id;
2347
2348 for_each_plane_id_on_crtc(crtc, plane_id) {
2349 struct skl_plane_wm *wm =
2350 &crtc_state->wm.skl.optimal.planes[plane_id];
2351
2352 wm->sagv.wm0.enable = false;
2353 wm->sagv.trans_wm.enable = false;
2354 }
2355 }
2356
2357 return 0;
2358}
2359
2360static int skl_build_pipe_wm(struct intel_atomic_state *state,
2361 struct intel_crtc *crtc)
2362{
2363 struct intel_display *display = to_intel_display(crtc);
2364 struct intel_crtc_state *crtc_state =
2365 intel_atomic_get_new_crtc_state(state, crtc);
2366 const struct intel_plane_state *plane_state;
2367 struct intel_plane *plane;
2368 int ret, i;
2369
2370 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2371 /*
2372 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2373 * instead but we don't populate that correctly for NV12 Y
2374 * planes so for now hack this.
2375 */
2376 if (plane->pipe != crtc->pipe)
2377 continue;
2378
2379 if (DISPLAY_VER(display) >= 11)
2380 ret = icl_build_plane_wm(crtc_state, plane_state);
2381 else
2382 ret = skl_build_plane_wm(crtc_state, plane_state);
2383 if (ret)
2384 return ret;
2385 }
2386
2387 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2388
2389 return skl_wm_check_vblank(crtc_state);
2390}
2391
2392static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2393 const struct skl_wm_level *l2)
2394{
2395 return l1->enable == l2->enable &&
2396 l1->ignore_lines == l2->ignore_lines &&
2397 l1->lines == l2->lines &&
2398 l1->blocks == l2->blocks &&
2399 l1->auto_min_alloc_wm_enable == l2->auto_min_alloc_wm_enable;
2400}
2401
2402static bool skl_plane_wm_equals(struct intel_display *display,
2403 const struct skl_plane_wm *wm1,
2404 const struct skl_plane_wm *wm2)
2405{
2406 int level;
2407
2408 for (level = 0; level < display->wm.num_levels; level++) {
2409 /*
2410 * We don't check uv_wm as the hardware doesn't actually
2411 * use it. It only gets used for calculating the required
2412 * ddb allocation.
2413 */
2414 if (!skl_wm_level_equals(l1: &wm1->wm[level], l2: &wm2->wm[level]))
2415 return false;
2416 }
2417
2418 return skl_wm_level_equals(l1: &wm1->trans_wm, l2: &wm2->trans_wm) &&
2419 skl_wm_level_equals(l1: &wm1->sagv.wm0, l2: &wm2->sagv.wm0) &&
2420 skl_wm_level_equals(l1: &wm1->sagv.trans_wm, l2: &wm2->sagv.trans_wm);
2421}
2422
2423static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2424 const struct skl_ddb_entry *b)
2425{
2426 return a->start < b->end && b->start < a->end;
2427}
2428
2429static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2430 const struct skl_ddb_entry *b)
2431{
2432 if (a->end && b->end) {
2433 a->start = min(a->start, b->start);
2434 a->end = max(a->end, b->end);
2435 } else if (b->end) {
2436 a->start = b->start;
2437 a->end = b->end;
2438 }
2439}
2440
2441bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2442 const struct skl_ddb_entry *entries,
2443 int num_entries, int ignore_idx)
2444{
2445 int i;
2446
2447 for (i = 0; i < num_entries; i++) {
2448 if (i != ignore_idx &&
2449 skl_ddb_entries_overlap(a: ddb, b: &entries[i]))
2450 return true;
2451 }
2452
2453 return false;
2454}
2455
2456static int
2457skl_ddb_add_affected_planes(struct intel_atomic_state *state,
2458 struct intel_crtc *crtc)
2459{
2460 struct intel_display *display = to_intel_display(state);
2461 const struct intel_crtc_state *old_crtc_state =
2462 intel_atomic_get_old_crtc_state(state, crtc);
2463 struct intel_crtc_state *new_crtc_state =
2464 intel_atomic_get_new_crtc_state(state, crtc);
2465 struct intel_plane *plane;
2466
2467 for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2468 struct intel_plane_state *plane_state;
2469 enum plane_id plane_id = plane->id;
2470
2471 if (skl_ddb_entry_equal(e1: &old_crtc_state->wm.skl.plane_ddb[plane_id],
2472 e2: &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2473 skl_ddb_entry_equal(e1: &old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2474 e2: &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2475 continue;
2476
2477 if (new_crtc_state->do_async_flip) {
2478 drm_dbg_kms(display->drm, "[PLANE:%d:%s] Can't change DDB during async flip\n",
2479 plane->base.base.id, plane->base.name);
2480 return -EINVAL;
2481 }
2482
2483 plane_state = intel_atomic_get_plane_state(state, plane);
2484 if (IS_ERR(ptr: plane_state))
2485 return PTR_ERR(ptr: plane_state);
2486
2487 new_crtc_state->update_planes |= BIT(plane_id);
2488 new_crtc_state->async_flip_planes = 0;
2489 new_crtc_state->do_async_flip = false;
2490 }
2491
2492 return 0;
2493}
2494
2495static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2496{
2497 struct intel_display *display = to_intel_display(dbuf_state->base.state->base.dev);
2498 u8 enabled_slices;
2499 enum pipe pipe;
2500
2501 /*
2502 * FIXME: For now we always enable slice S1 as per
2503 * the Bspec display initialization sequence.
2504 */
2505 enabled_slices = BIT(DBUF_S1);
2506
2507 for_each_pipe(display, pipe)
2508 enabled_slices |= dbuf_state->slices[pipe];
2509
2510 return enabled_slices;
2511}
2512
2513static int
2514skl_compute_ddb(struct intel_atomic_state *state)
2515{
2516 struct intel_display *display = to_intel_display(state);
2517 const struct intel_dbuf_state *old_dbuf_state;
2518 struct intel_dbuf_state *new_dbuf_state = NULL;
2519 struct intel_crtc_state *new_crtc_state;
2520 struct intel_crtc *crtc;
2521 int ret, i;
2522
2523 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2524 new_dbuf_state = intel_atomic_get_dbuf_state(state);
2525 if (IS_ERR(ptr: new_dbuf_state))
2526 return PTR_ERR(ptr: new_dbuf_state);
2527
2528 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2529 break;
2530 }
2531
2532 if (!new_dbuf_state)
2533 return 0;
2534
2535 new_dbuf_state->active_pipes =
2536 intel_calc_active_pipes(state, active_pipes: old_dbuf_state->active_pipes);
2537
2538 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2539 ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2540 if (ret)
2541 return ret;
2542 }
2543
2544 if (HAS_MBUS_JOINING(display)) {
2545 new_dbuf_state->joined_mbus =
2546 adlp_check_mbus_joined(active_pipes: new_dbuf_state->active_pipes);
2547
2548 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2549 ret = intel_cdclk_state_set_joined_mbus(state, joined_mbus: new_dbuf_state->joined_mbus);
2550 if (ret)
2551 return ret;
2552 }
2553 }
2554
2555 for_each_intel_crtc(display->drm, crtc) {
2556 enum pipe pipe = crtc->pipe;
2557
2558 new_dbuf_state->slices[pipe] =
2559 skl_compute_dbuf_slices(crtc, active_pipes: new_dbuf_state->active_pipes,
2560 join_mbus: new_dbuf_state->joined_mbus);
2561
2562 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2563 continue;
2564
2565 ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2566 if (ret)
2567 return ret;
2568 }
2569
2570 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(dbuf_state: new_dbuf_state);
2571
2572 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2573 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2574 ret = intel_atomic_serialize_global_state(obj_state: &new_dbuf_state->base);
2575 if (ret)
2576 return ret;
2577
2578 drm_dbg_kms(display->drm,
2579 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2580 old_dbuf_state->enabled_slices,
2581 new_dbuf_state->enabled_slices,
2582 DISPLAY_INFO(display)->dbuf.slice_mask,
2583 str_yes_no(old_dbuf_state->joined_mbus),
2584 str_yes_no(new_dbuf_state->joined_mbus));
2585 }
2586
2587 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2588 enum pipe pipe = crtc->pipe;
2589
2590 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state: new_crtc_state);
2591
2592 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2593 continue;
2594
2595 ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2596 if (ret)
2597 return ret;
2598 }
2599
2600 for_each_intel_crtc(display->drm, crtc) {
2601 ret = skl_crtc_allocate_ddb(state, crtc);
2602 if (ret)
2603 return ret;
2604 }
2605
2606 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2607 ret = skl_crtc_allocate_plane_ddb(state, crtc);
2608 if (ret)
2609 return ret;
2610
2611 ret = skl_ddb_add_affected_planes(state, crtc);
2612 if (ret)
2613 return ret;
2614 }
2615
2616 return 0;
2617}
2618
2619static char enast(bool enable)
2620{
2621 return enable ? '*' : ' ';
2622}
2623
2624static noinline_for_stack void
2625skl_print_plane_changes(struct intel_display *display,
2626 struct intel_plane *plane,
2627 const struct skl_plane_wm *old_wm,
2628 const struct skl_plane_wm *new_wm)
2629{
2630 drm_dbg_kms(display->drm,
2631 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2632 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2633 plane->base.base.id, plane->base.name,
2634 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2635 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2636 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2637 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2638 enast(old_wm->trans_wm.enable),
2639 enast(old_wm->sagv.wm0.enable),
2640 enast(old_wm->sagv.trans_wm.enable),
2641 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2642 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2643 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2644 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2645 enast(new_wm->trans_wm.enable),
2646 enast(new_wm->sagv.wm0.enable),
2647 enast(new_wm->sagv.trans_wm.enable));
2648
2649 drm_dbg_kms(display->drm,
2650 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2651 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2652 plane->base.base.id, plane->base.name,
2653 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2654 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2655 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2656 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2657 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2658 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2659 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2660 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2661 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2662 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2663 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2664 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2665 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2666 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2667 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2668 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2669 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2670 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2671 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2672 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2673 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2674 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2675
2676 drm_dbg_kms(display->drm,
2677 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2678 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2679 plane->base.base.id, plane->base.name,
2680 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2681 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2682 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2683 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2684 old_wm->trans_wm.blocks,
2685 old_wm->sagv.wm0.blocks,
2686 old_wm->sagv.trans_wm.blocks,
2687 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2688 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2689 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2690 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2691 new_wm->trans_wm.blocks,
2692 new_wm->sagv.wm0.blocks,
2693 new_wm->sagv.trans_wm.blocks);
2694
2695 drm_dbg_kms(display->drm,
2696 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2697 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2698 plane->base.base.id, plane->base.name,
2699 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2700 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2701 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2702 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2703 old_wm->trans_wm.min_ddb_alloc,
2704 old_wm->sagv.wm0.min_ddb_alloc,
2705 old_wm->sagv.trans_wm.min_ddb_alloc,
2706 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2707 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2708 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2709 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2710 new_wm->trans_wm.min_ddb_alloc,
2711 new_wm->sagv.wm0.min_ddb_alloc,
2712 new_wm->sagv.trans_wm.min_ddb_alloc);
2713}
2714
2715static void
2716skl_print_wm_changes(struct intel_atomic_state *state)
2717{
2718 struct intel_display *display = to_intel_display(state);
2719 const struct intel_crtc_state *old_crtc_state;
2720 const struct intel_crtc_state *new_crtc_state;
2721 struct intel_plane *plane;
2722 struct intel_crtc *crtc;
2723 int i;
2724
2725 if (!drm_debug_enabled(DRM_UT_KMS))
2726 return;
2727
2728 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2729 new_crtc_state, i) {
2730 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2731
2732 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2733 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2734
2735 for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2736 enum plane_id plane_id = plane->id;
2737 const struct skl_ddb_entry *old, *new;
2738
2739 old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2740 new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2741
2742 if (skl_ddb_entry_equal(e1: old, e2: new))
2743 continue;
2744 drm_dbg_kms(display->drm,
2745 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2746 plane->base.base.id, plane->base.name,
2747 old->start, old->end, new->start, new->end,
2748 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2749 }
2750
2751 for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2752 enum plane_id plane_id = plane->id;
2753 const struct skl_plane_wm *old_wm, *new_wm;
2754
2755 old_wm = &old_pipe_wm->planes[plane_id];
2756 new_wm = &new_pipe_wm->planes[plane_id];
2757
2758 if (skl_plane_wm_equals(display, wm1: old_wm, wm2: new_wm))
2759 continue;
2760
2761 skl_print_plane_changes(display, plane, old_wm, new_wm);
2762 }
2763 }
2764}
2765
2766static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2767 const struct skl_pipe_wm *old_pipe_wm,
2768 const struct skl_pipe_wm *new_pipe_wm)
2769{
2770 struct intel_display *display = to_intel_display(plane);
2771 int level;
2772
2773 for (level = 0; level < display->wm.num_levels; level++) {
2774 /*
2775 * We don't check uv_wm as the hardware doesn't actually
2776 * use it. It only gets used for calculating the required
2777 * ddb allocation.
2778 */
2779 if (!skl_wm_level_equals(l1: skl_plane_wm_level(pipe_wm: old_pipe_wm, plane_id: plane->id, level),
2780 l2: skl_plane_wm_level(pipe_wm: new_pipe_wm, plane_id: plane->id, level)))
2781 return false;
2782 }
2783
2784 if (HAS_HW_SAGV_WM(display)) {
2785 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2786 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2787
2788 if (!skl_wm_level_equals(l1: &old_wm->sagv.wm0, l2: &new_wm->sagv.wm0) ||
2789 !skl_wm_level_equals(l1: &old_wm->sagv.trans_wm, l2: &new_wm->sagv.trans_wm))
2790 return false;
2791 }
2792
2793 return skl_wm_level_equals(l1: skl_plane_trans_wm(pipe_wm: old_pipe_wm, plane_id: plane->id),
2794 l2: skl_plane_trans_wm(pipe_wm: new_pipe_wm, plane_id: plane->id));
2795}
2796
2797/*
2798 * To make sure the cursor watermark registers are always consistent
2799 * with our computed state the following scenario needs special
2800 * treatment:
2801 *
2802 * 1. enable cursor
2803 * 2. move cursor entirely offscreen
2804 * 3. disable cursor
2805 *
2806 * Step 2. does call .disable_plane() but does not zero the watermarks
2807 * (since we consider an offscreen cursor still active for the purposes
2808 * of watermarks). Step 3. would not normally call .disable_plane()
2809 * because the actual plane visibility isn't changing, and we don't
2810 * deallocate the cursor ddb until the pipe gets disabled. So we must
2811 * force step 3. to call .disable_plane() to update the watermark
2812 * registers properly.
2813 *
2814 * Other planes do not suffer from this issues as their watermarks are
2815 * calculated based on the actual plane visibility. The only time this
2816 * can trigger for the other planes is during the initial readout as the
2817 * default value of the watermarks registers is not zero.
2818 */
2819static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2820 struct intel_crtc *crtc)
2821{
2822 struct intel_display *display = to_intel_display(state);
2823 const struct intel_crtc_state *old_crtc_state =
2824 intel_atomic_get_old_crtc_state(state, crtc);
2825 struct intel_crtc_state *new_crtc_state =
2826 intel_atomic_get_new_crtc_state(state, crtc);
2827 struct intel_plane *plane;
2828
2829 for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2830 struct intel_plane_state *plane_state;
2831 enum plane_id plane_id = plane->id;
2832
2833 /*
2834 * Force a full wm update for every plane on modeset.
2835 * Required because the reset value of the wm registers
2836 * is non-zero, whereas we want all disabled planes to
2837 * have zero watermarks. So if we turn off the relevant
2838 * power well the hardware state will go out of sync
2839 * with the software state.
2840 */
2841 if (!intel_crtc_needs_modeset(crtc_state: new_crtc_state) &&
2842 skl_plane_selected_wm_equals(plane,
2843 old_pipe_wm: &old_crtc_state->wm.skl.optimal,
2844 new_pipe_wm: &new_crtc_state->wm.skl.optimal))
2845 continue;
2846
2847 if (new_crtc_state->do_async_flip) {
2848 drm_dbg_kms(display->drm, "[PLANE:%d:%s] Can't change watermarks during async flip\n",
2849 plane->base.base.id, plane->base.name);
2850 return -EINVAL;
2851 }
2852
2853 plane_state = intel_atomic_get_plane_state(state, plane);
2854 if (IS_ERR(ptr: plane_state))
2855 return PTR_ERR(ptr: plane_state);
2856
2857 new_crtc_state->update_planes |= BIT(plane_id);
2858 new_crtc_state->async_flip_planes = 0;
2859 new_crtc_state->do_async_flip = false;
2860 }
2861
2862 return 0;
2863}
2864
2865static int pkgc_max_linetime(struct intel_atomic_state *state)
2866{
2867 struct intel_display *display = to_intel_display(state);
2868 const struct intel_crtc_state *crtc_state;
2869 struct intel_crtc *crtc;
2870 int i, max_linetime;
2871
2872 /*
2873 * Apparenty the hardware uses WM_LINETIME internally for
2874 * this stuff, compute everything based on that.
2875 */
2876 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2877 display->pkgc.disable[crtc->pipe] = crtc_state->vrr.enable;
2878 display->pkgc.linetime[crtc->pipe] = DIV_ROUND_UP(crtc_state->linetime, 8);
2879 }
2880
2881 max_linetime = 0;
2882 for_each_intel_crtc(display->drm, crtc) {
2883 if (display->pkgc.disable[crtc->pipe])
2884 return 0;
2885
2886 max_linetime = max(display->pkgc.linetime[crtc->pipe], max_linetime);
2887 }
2888
2889 return max_linetime;
2890}
2891
2892void
2893intel_program_dpkgc_latency(struct intel_atomic_state *state)
2894{
2895 struct intel_display *display = to_intel_display(state);
2896 int max_linetime, latency, added_wake_time = 0;
2897
2898 if (DISPLAY_VER(display) < 20)
2899 return;
2900
2901 mutex_lock(lock: &display->wm.wm_mutex);
2902
2903 latency = skl_watermark_max_latency(display, initial_wm_level: 1);
2904
2905 /* FIXME runtime changes to enable_flipq are racy */
2906 if (display->params.enable_flipq)
2907 added_wake_time = intel_flipq_exec_time_us(display);
2908
2909 /*
2910 * Wa_22020432604
2911 * "PKG_C_LATENCY Added Wake Time field is not working"
2912 */
2913 if (latency && IS_DISPLAY_VER(display, 20, 30)) {
2914 latency += added_wake_time;
2915 added_wake_time = 0;
2916 }
2917
2918 max_linetime = pkgc_max_linetime(state);
2919
2920 if (max_linetime == 0 || latency == 0) {
2921 latency = REG_FIELD_GET(LNL_PKG_C_LATENCY_MASK,
2922 LNL_PKG_C_LATENCY_MASK);
2923 added_wake_time = 0;
2924 } else {
2925 /*
2926 * Wa_22020299601
2927 * "Increase the latency programmed in PKG_C_LATENCY Pkg C Latency to be a
2928 * multiple of the pipeline time from WM_LINETIME"
2929 */
2930 latency = roundup(latency, max_linetime);
2931 }
2932
2933 intel_de_write(display, LNL_PKG_C_LATENCY,
2934 REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time) |
2935 REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, latency));
2936
2937 mutex_unlock(lock: &display->wm.wm_mutex);
2938}
2939
2940static int
2941skl_compute_wm(struct intel_atomic_state *state)
2942{
2943 struct intel_display *display = to_intel_display(state);
2944 struct intel_crtc *crtc;
2945 struct intel_crtc_state __maybe_unused *new_crtc_state;
2946 int ret, i;
2947
2948 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2949 ret = skl_build_pipe_wm(state, crtc);
2950 if (ret)
2951 return ret;
2952 }
2953
2954 ret = skl_compute_ddb(state);
2955 if (ret)
2956 return ret;
2957
2958 /*
2959 * skl_compute_ddb() will have adjusted the final watermarks
2960 * based on how much ddb is available. Now we can actually
2961 * check if the final watermarks changed.
2962 */
2963 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2964 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
2965
2966 /*
2967 * We store use_sagv_wm in the crtc state rather than relying on
2968 * that bw state since we have no convenient way to get at the
2969 * latter from the plane commit hooks (especially in the legacy
2970 * cursor case).
2971 *
2972 * drm_atomic_check_only() gets upset if we pull more crtcs
2973 * into the state, so we have to calculate this based on the
2974 * individual intel_crtc_can_enable_sagv() rather than
2975 * the overall intel_bw_can_enable_sagv(). Otherwise the
2976 * crtcs not included in the commit would not switch to the
2977 * SAGV watermarks when we are about to enable SAGV, and that
2978 * would lead to underruns. This does mean extra power draw
2979 * when only a subset of the crtcs are blocking SAGV as the
2980 * other crtcs can't be allowed to use the more optimal
2981 * normal (ie. non-SAGV) watermarks.
2982 */
2983 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(display) &&
2984 DISPLAY_VER(display) >= 12 &&
2985 intel_crtc_can_enable_sagv(crtc_state: new_crtc_state);
2986
2987 ret = skl_wm_add_affected_planes(state, crtc);
2988 if (ret)
2989 return ret;
2990 }
2991
2992 skl_print_wm_changes(state);
2993
2994 return 0;
2995}
2996
2997static void skl_wm_level_from_reg_val(struct intel_display *display,
2998 u32 val, struct skl_wm_level *level)
2999{
3000 level->enable = val & PLANE_WM_EN;
3001 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
3002 level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
3003 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
3004 level->auto_min_alloc_wm_enable = DISPLAY_VER(display) >= 30 ?
3005 val & PLANE_WM_AUTO_MIN_ALLOC_EN : 0;
3006}
3007
3008static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
3009 struct skl_pipe_wm *out)
3010{
3011 struct intel_display *display = to_intel_display(crtc);
3012 enum pipe pipe = crtc->pipe;
3013 enum plane_id plane_id;
3014 int level;
3015 u32 val;
3016
3017 for_each_plane_id_on_crtc(crtc, plane_id) {
3018 struct skl_plane_wm *wm = &out->planes[plane_id];
3019
3020 for (level = 0; level < display->wm.num_levels; level++) {
3021 if (plane_id != PLANE_CURSOR)
3022 val = intel_de_read(display, PLANE_WM(pipe, plane_id, level));
3023 else
3024 val = intel_de_read(display, CUR_WM(pipe, level));
3025
3026 skl_wm_level_from_reg_val(display, val, level: &wm->wm[level]);
3027 }
3028
3029 if (plane_id != PLANE_CURSOR)
3030 val = intel_de_read(display, PLANE_WM_TRANS(pipe, plane_id));
3031 else
3032 val = intel_de_read(display, CUR_WM_TRANS(pipe));
3033
3034 skl_wm_level_from_reg_val(display, val, level: &wm->trans_wm);
3035
3036 if (HAS_HW_SAGV_WM(display)) {
3037 if (plane_id != PLANE_CURSOR)
3038 val = intel_de_read(display, PLANE_WM_SAGV(pipe, plane_id));
3039 else
3040 val = intel_de_read(display, CUR_WM_SAGV(pipe));
3041
3042 skl_wm_level_from_reg_val(display, val, level: &wm->sagv.wm0);
3043
3044 if (plane_id != PLANE_CURSOR)
3045 val = intel_de_read(display, PLANE_WM_SAGV_TRANS(pipe, plane_id));
3046 else
3047 val = intel_de_read(display, CUR_WM_SAGV_TRANS(pipe));
3048
3049 skl_wm_level_from_reg_val(display, val, level: &wm->sagv.trans_wm);
3050 } else if (DISPLAY_VER(display) >= 12) {
3051 wm->sagv.wm0 = wm->wm[0];
3052 wm->sagv.trans_wm = wm->trans_wm;
3053 }
3054 }
3055}
3056
3057static void skl_wm_get_hw_state(struct intel_display *display)
3058{
3059 struct intel_dbuf_state *dbuf_state =
3060 to_intel_dbuf_state(display->dbuf.obj.state);
3061 struct intel_crtc *crtc;
3062
3063 if (HAS_MBUS_JOINING(display))
3064 dbuf_state->joined_mbus = intel_de_read(display, MBUS_CTL) & MBUS_JOIN;
3065
3066 dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(display, cdclk_config: &display->cdclk.hw);
3067 dbuf_state->active_pipes = 0;
3068
3069 for_each_intel_crtc(display->drm, crtc) {
3070 struct intel_crtc_state *crtc_state =
3071 to_intel_crtc_state(crtc->base.state);
3072 enum pipe pipe = crtc->pipe;
3073 unsigned int mbus_offset;
3074 enum plane_id plane_id;
3075 u8 slices;
3076
3077 memset(s: &crtc_state->wm.skl.optimal, c: 0,
3078 n: sizeof(crtc_state->wm.skl.optimal));
3079 if (crtc_state->hw.active) {
3080 skl_pipe_wm_get_hw_state(crtc, out: &crtc_state->wm.skl.optimal);
3081 dbuf_state->active_pipes |= BIT(pipe);
3082 }
3083 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3084
3085 memset(s: &dbuf_state->ddb[pipe], c: 0, n: sizeof(dbuf_state->ddb[pipe]));
3086
3087 for_each_plane_id_on_crtc(crtc, plane_id) {
3088 struct skl_ddb_entry *ddb =
3089 &crtc_state->wm.skl.plane_ddb[plane_id];
3090 struct skl_ddb_entry *ddb_y =
3091 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3092 u16 *min_ddb =
3093 &crtc_state->wm.skl.plane_min_ddb[plane_id];
3094 u16 *interim_ddb =
3095 &crtc_state->wm.skl.plane_interim_ddb[plane_id];
3096
3097 if (!crtc_state->hw.active)
3098 continue;
3099
3100 skl_ddb_get_hw_plane_state(display, pipe: crtc->pipe,
3101 plane_id, ddb, ddb_y,
3102 min_ddb, interim_ddb);
3103
3104 skl_ddb_entry_union(a: &dbuf_state->ddb[pipe], b: ddb);
3105 skl_ddb_entry_union(a: &dbuf_state->ddb[pipe], b: ddb_y);
3106 }
3107
3108 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3109
3110 /*
3111 * Used for checking overlaps, so we need absolute
3112 * offsets instead of MBUS relative offsets.
3113 */
3114 slices = skl_compute_dbuf_slices(crtc, active_pipes: dbuf_state->active_pipes,
3115 join_mbus: dbuf_state->joined_mbus);
3116 mbus_offset = mbus_ddb_offset(display, slice_mask: slices);
3117 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3118 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3119
3120 /* The slices actually used by the planes on the pipe */
3121 dbuf_state->slices[pipe] =
3122 skl_ddb_dbuf_slice_mask(display, entry: &crtc_state->wm.skl.ddb);
3123
3124 drm_dbg_kms(display->drm,
3125 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3126 crtc->base.base.id, crtc->base.name,
3127 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3128 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3129 str_yes_no(dbuf_state->joined_mbus));
3130 }
3131
3132 dbuf_state->enabled_slices = display->dbuf.enabled_slices;
3133}
3134
3135bool skl_watermark_ipc_enabled(struct intel_display *display)
3136{
3137 return display->wm.ipc_enabled;
3138}
3139
3140void skl_watermark_ipc_update(struct intel_display *display)
3141{
3142 if (!HAS_IPC(display))
3143 return;
3144
3145 intel_de_rmw(display, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3146 set: skl_watermark_ipc_enabled(display) ? DISP_IPC_ENABLE : 0);
3147}
3148
3149static bool skl_watermark_ipc_can_enable(struct intel_display *display)
3150{
3151 /* Display WA #0477 WaDisableIPC: skl */
3152 if (display->platform.skylake)
3153 return false;
3154
3155 /* Display WA #1141: SKL:all KBL:all CFL */
3156 if (display->platform.kabylake ||
3157 display->platform.coffeelake ||
3158 display->platform.cometlake) {
3159 const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
3160
3161 return dram_info->symmetric_memory;
3162 }
3163
3164 return true;
3165}
3166
3167void skl_watermark_ipc_init(struct intel_display *display)
3168{
3169 if (!HAS_IPC(display))
3170 return;
3171
3172 display->wm.ipc_enabled = skl_watermark_ipc_can_enable(display);
3173
3174 skl_watermark_ipc_update(display);
3175}
3176
3177static void
3178adjust_wm_latency(struct intel_display *display,
3179 u16 wm[], int num_levels, int read_latency)
3180{
3181 const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
3182 int i, level;
3183
3184 /*
3185 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3186 * need to be disabled. We make sure to sanitize the values out
3187 * of the punit to satisfy this requirement.
3188 */
3189 for (level = 1; level < num_levels; level++) {
3190 if (wm[level] == 0) {
3191 for (i = level + 1; i < num_levels; i++)
3192 wm[i] = 0;
3193
3194 num_levels = level;
3195 break;
3196 }
3197 }
3198
3199 /*
3200 * WaWmMemoryReadLatency
3201 *
3202 * punit doesn't take into account the read latency so we need
3203 * to add proper adjustment to each valid level we retrieve
3204 * from the punit when level 0 response data is 0us.
3205 */
3206 if (wm[0] == 0) {
3207 for (level = 0; level < num_levels; level++)
3208 wm[level] += read_latency;
3209 }
3210
3211 /*
3212 * WA Level-0 adjustment for 16Gb DIMMs: SKL+
3213 * If we could not get dimm info enable this WA to prevent from
3214 * any underrun. If not able to get DIMM info assume 16Gb DIMM
3215 * to avoid any underrun.
3216 */
3217 if (!display->platform.dg2 && dram_info->has_16gb_dimms)
3218 wm[0] += 1;
3219}
3220
3221static void mtl_read_wm_latency(struct intel_display *display, u16 wm[])
3222{
3223 int num_levels = display->wm.num_levels;
3224 u32 val;
3225
3226 val = intel_de_read(display, MTL_LATENCY_LP0_LP1);
3227 wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3228 wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3229
3230 val = intel_de_read(display, MTL_LATENCY_LP2_LP3);
3231 wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3232 wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3233
3234 val = intel_de_read(display, MTL_LATENCY_LP4_LP5);
3235 wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3236 wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3237
3238 adjust_wm_latency(display, wm, num_levels, read_latency: 6);
3239}
3240
3241static void skl_read_wm_latency(struct intel_display *display, u16 wm[])
3242{
3243 int num_levels = display->wm.num_levels;
3244 int read_latency = DISPLAY_VER(display) >= 12 ? 3 : 2;
3245 int mult = display->platform.dg2 ? 2 : 1;
3246 u32 val;
3247 int ret;
3248
3249 /* read the first set of memory latencies[0:3] */
3250 val = 0; /* data0 to be programmed to 0 for first set */
3251 ret = intel_pcode_read(drm: display->drm, GEN9_PCODE_READ_MEM_LATENCY, val: &val, NULL);
3252 if (ret) {
3253 drm_err(display->drm, "SKL Mailbox read error = %d\n", ret);
3254 return;
3255 }
3256
3257 wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3258 wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3259 wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3260 wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3261
3262 /* read the second set of memory latencies[4:7] */
3263 val = 1; /* data0 to be programmed to 1 for second set */
3264 ret = intel_pcode_read(drm: display->drm, GEN9_PCODE_READ_MEM_LATENCY, val: &val, NULL);
3265 if (ret) {
3266 drm_err(display->drm, "SKL Mailbox read error = %d\n", ret);
3267 return;
3268 }
3269
3270 wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3271 wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3272 wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3273 wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3274
3275 adjust_wm_latency(display, wm, num_levels, read_latency);
3276}
3277
3278static void skl_setup_wm_latency(struct intel_display *display)
3279{
3280 if (HAS_HW_SAGV_WM(display))
3281 display->wm.num_levels = 6;
3282 else
3283 display->wm.num_levels = 8;
3284
3285 if (DISPLAY_VER(display) >= 14)
3286 mtl_read_wm_latency(display, wm: display->wm.skl_latency);
3287 else
3288 skl_read_wm_latency(display, wm: display->wm.skl_latency);
3289
3290 intel_print_wm_latency(display, name: "Gen9 Plane", wm: display->wm.skl_latency);
3291}
3292
3293static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3294{
3295 struct intel_dbuf_state *dbuf_state;
3296
3297 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3298 if (!dbuf_state)
3299 return NULL;
3300
3301 return &dbuf_state->base;
3302}
3303
3304static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3305 struct intel_global_state *state)
3306{
3307 kfree(objp: state);
3308}
3309
3310static const struct intel_global_state_funcs intel_dbuf_funcs = {
3311 .atomic_duplicate_state = intel_dbuf_duplicate_state,
3312 .atomic_destroy_state = intel_dbuf_destroy_state,
3313};
3314
3315struct intel_dbuf_state *
3316intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3317{
3318 struct intel_display *display = to_intel_display(state);
3319 struct intel_global_state *dbuf_state;
3320
3321 dbuf_state = intel_atomic_get_global_obj_state(state, obj: &display->dbuf.obj);
3322 if (IS_ERR(ptr: dbuf_state))
3323 return ERR_CAST(ptr: dbuf_state);
3324
3325 return to_intel_dbuf_state(dbuf_state);
3326}
3327
3328int intel_dbuf_init(struct intel_display *display)
3329{
3330 struct intel_dbuf_state *dbuf_state;
3331
3332 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3333 if (!dbuf_state)
3334 return -ENOMEM;
3335
3336 intel_atomic_global_obj_init(display, obj: &display->dbuf.obj,
3337 state: &dbuf_state->base, funcs: &intel_dbuf_funcs);
3338
3339 return 0;
3340}
3341
3342static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3343{
3344 switch (pipe) {
3345 case PIPE_A:
3346 case PIPE_D:
3347 active_pipes &= BIT(PIPE_A) | BIT(PIPE_D);
3348 break;
3349 case PIPE_B:
3350 case PIPE_C:
3351 active_pipes &= BIT(PIPE_B) | BIT(PIPE_C);
3352 break;
3353 default: /* to suppress compiler warning */
3354 MISSING_CASE(pipe);
3355 return false;
3356 }
3357
3358 return is_power_of_2(n: active_pipes);
3359}
3360
3361static u32 pipe_mbus_dbox_ctl(const struct intel_crtc *crtc,
3362 const struct intel_dbuf_state *dbuf_state)
3363{
3364 struct intel_display *display = to_intel_display(crtc);
3365 u32 val = 0;
3366
3367 if (DISPLAY_VER(display) >= 14)
3368 val |= MBUS_DBOX_I_CREDIT(2);
3369
3370 if (DISPLAY_VER(display) >= 12) {
3371 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3372 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3373 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3374 }
3375
3376 if (DISPLAY_VER(display) >= 14)
3377 val |= dbuf_state->joined_mbus ?
3378 MBUS_DBOX_A_CREDIT(12) : MBUS_DBOX_A_CREDIT(8);
3379 else if (display->platform.alderlake_p)
3380 /* Wa_22010947358:adl-p */
3381 val |= dbuf_state->joined_mbus ?
3382 MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4);
3383 else
3384 val |= MBUS_DBOX_A_CREDIT(2);
3385
3386 if (DISPLAY_VER(display) >= 14) {
3387 val |= MBUS_DBOX_B_CREDIT(0xA);
3388 } else if (display->platform.alderlake_p) {
3389 val |= MBUS_DBOX_BW_CREDIT(2);
3390 val |= MBUS_DBOX_B_CREDIT(8);
3391 } else if (DISPLAY_VER(display) >= 12) {
3392 val |= MBUS_DBOX_BW_CREDIT(2);
3393 val |= MBUS_DBOX_B_CREDIT(12);
3394 } else {
3395 val |= MBUS_DBOX_BW_CREDIT(1);
3396 val |= MBUS_DBOX_B_CREDIT(8);
3397 }
3398
3399 if (DISPLAY_VERx100(display) == 1400) {
3400 if (xelpdp_is_only_pipe_per_dbuf_bank(pipe: crtc->pipe, active_pipes: dbuf_state->active_pipes))
3401 val |= MBUS_DBOX_BW_8CREDITS_MTL;
3402 else
3403 val |= MBUS_DBOX_BW_4CREDITS_MTL;
3404 }
3405
3406 return val;
3407}
3408
3409static void pipe_mbus_dbox_ctl_update(struct intel_display *display,
3410 const struct intel_dbuf_state *dbuf_state)
3411{
3412 struct intel_crtc *crtc;
3413
3414 for_each_intel_crtc_in_pipe_mask(display->drm, crtc, dbuf_state->active_pipes)
3415 intel_de_write(display, PIPE_MBUS_DBOX_CTL(crtc->pipe),
3416 val: pipe_mbus_dbox_ctl(crtc, dbuf_state));
3417}
3418
3419static void intel_mbus_dbox_update(struct intel_atomic_state *state)
3420{
3421 struct intel_display *display = to_intel_display(state);
3422 const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3423
3424 if (DISPLAY_VER(display) < 11)
3425 return;
3426
3427 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3428 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3429 if (!new_dbuf_state ||
3430 (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3431 new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3432 return;
3433
3434 pipe_mbus_dbox_ctl_update(display, dbuf_state: new_dbuf_state);
3435}
3436
3437int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
3438 int ratio)
3439{
3440 struct intel_dbuf_state *dbuf_state;
3441
3442 dbuf_state = intel_atomic_get_dbuf_state(state);
3443 if (IS_ERR(ptr: dbuf_state))
3444 return PTR_ERR(ptr: dbuf_state);
3445
3446 dbuf_state->mdclk_cdclk_ratio = ratio;
3447
3448 return intel_atomic_lock_global_state(obj_state: &dbuf_state->base);
3449}
3450
3451void intel_dbuf_mdclk_cdclk_ratio_update(struct intel_display *display,
3452 int ratio, bool joined_mbus)
3453{
3454 enum dbuf_slice slice;
3455
3456 if (!HAS_MBUS_JOINING(display))
3457 return;
3458
3459 if (DISPLAY_VER(display) >= 20)
3460 intel_de_rmw(display, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK,
3461 MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
3462
3463 if (joined_mbus)
3464 ratio *= 2;
3465
3466 drm_dbg_kms(display->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n",
3467 ratio, str_yes_no(joined_mbus));
3468
3469 for_each_dbuf_slice(display, slice)
3470 intel_de_rmw(display, DBUF_CTL_S(slice),
3471 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3472 DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
3473}
3474
3475static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state)
3476{
3477 struct intel_display *display = to_intel_display(state);
3478 const struct intel_dbuf_state *old_dbuf_state =
3479 intel_atomic_get_old_dbuf_state(state);
3480 const struct intel_dbuf_state *new_dbuf_state =
3481 intel_atomic_get_new_dbuf_state(state);
3482 int mdclk_cdclk_ratio;
3483
3484 if (intel_cdclk_is_decreasing_later(state)) {
3485 /* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */
3486 mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio;
3487 } else {
3488 /* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */
3489 mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio;
3490 }
3491
3492 intel_dbuf_mdclk_cdclk_ratio_update(display, ratio: mdclk_cdclk_ratio,
3493 joined_mbus: new_dbuf_state->joined_mbus);
3494}
3495
3496static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state,
3497 const struct intel_dbuf_state *dbuf_state)
3498{
3499 struct intel_display *display = to_intel_display(state);
3500 enum pipe pipe = ffs(dbuf_state->active_pipes) - 1;
3501 const struct intel_crtc_state *new_crtc_state;
3502 struct intel_crtc *crtc;
3503
3504 drm_WARN_ON(display->drm, !dbuf_state->joined_mbus);
3505 drm_WARN_ON(display->drm, !is_power_of_2(dbuf_state->active_pipes));
3506
3507 crtc = intel_crtc_for_pipe(display, pipe);
3508 new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
3509
3510 if (new_crtc_state && !intel_crtc_needs_modeset(crtc_state: new_crtc_state))
3511 return pipe;
3512 else
3513 return INVALID_PIPE;
3514}
3515
3516static void mbus_ctl_join_update(struct intel_display *display,
3517 const struct intel_dbuf_state *dbuf_state,
3518 enum pipe pipe)
3519{
3520 u32 mbus_ctl;
3521
3522 if (dbuf_state->joined_mbus)
3523 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN;
3524 else
3525 mbus_ctl = MBUS_HASHING_MODE_2x2;
3526
3527 if (pipe != INVALID_PIPE)
3528 mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe);
3529 else
3530 mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE;
3531
3532 intel_de_rmw(display, MBUS_CTL,
3533 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3534 MBUS_JOIN_PIPE_SELECT_MASK, set: mbus_ctl);
3535}
3536
3537static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state,
3538 enum pipe pipe)
3539{
3540 struct intel_display *display = to_intel_display(state);
3541 const struct intel_dbuf_state *old_dbuf_state =
3542 intel_atomic_get_old_dbuf_state(state);
3543 const struct intel_dbuf_state *new_dbuf_state =
3544 intel_atomic_get_new_dbuf_state(state);
3545
3546 drm_dbg_kms(display->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n",
3547 str_yes_no(old_dbuf_state->joined_mbus),
3548 str_yes_no(new_dbuf_state->joined_mbus),
3549 pipe != INVALID_PIPE ? pipe_name(pipe) : '*');
3550
3551 mbus_ctl_join_update(display, dbuf_state: new_dbuf_state, pipe);
3552}
3553
3554void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state)
3555{
3556 const struct intel_dbuf_state *new_dbuf_state =
3557 intel_atomic_get_new_dbuf_state(state);
3558 const struct intel_dbuf_state *old_dbuf_state =
3559 intel_atomic_get_old_dbuf_state(state);
3560
3561 if (!new_dbuf_state)
3562 return;
3563
3564 if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) {
3565 enum pipe pipe = intel_mbus_joined_pipe(state, dbuf_state: new_dbuf_state);
3566
3567 WARN_ON(!new_dbuf_state->base.changed);
3568
3569 intel_dbuf_mbus_join_update(state, pipe);
3570 intel_mbus_dbox_update(state);
3571 intel_dbuf_mdclk_min_tracker_update(state);
3572 }
3573}
3574
3575void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state)
3576{
3577 struct intel_display *display = to_intel_display(state);
3578 const struct intel_dbuf_state *new_dbuf_state =
3579 intel_atomic_get_new_dbuf_state(state);
3580 const struct intel_dbuf_state *old_dbuf_state =
3581 intel_atomic_get_old_dbuf_state(state);
3582
3583 if (!new_dbuf_state)
3584 return;
3585
3586 if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) {
3587 enum pipe pipe = intel_mbus_joined_pipe(state, dbuf_state: old_dbuf_state);
3588
3589 WARN_ON(!new_dbuf_state->base.changed);
3590
3591 intel_dbuf_mdclk_min_tracker_update(state);
3592 intel_mbus_dbox_update(state);
3593 intel_dbuf_mbus_join_update(state, pipe);
3594
3595 if (pipe != INVALID_PIPE) {
3596 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
3597
3598 intel_crtc_wait_for_next_vblank(crtc);
3599 }
3600 } else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus &&
3601 old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
3602 WARN_ON(!new_dbuf_state->base.changed);
3603
3604 intel_dbuf_mdclk_min_tracker_update(state);
3605 intel_mbus_dbox_update(state);
3606 }
3607
3608}
3609
3610void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3611{
3612 struct intel_display *display = to_intel_display(state);
3613 const struct intel_dbuf_state *new_dbuf_state =
3614 intel_atomic_get_new_dbuf_state(state);
3615 const struct intel_dbuf_state *old_dbuf_state =
3616 intel_atomic_get_old_dbuf_state(state);
3617 u8 old_slices, new_slices;
3618
3619 if (!new_dbuf_state)
3620 return;
3621
3622 old_slices = old_dbuf_state->enabled_slices;
3623 new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3624
3625 if (old_slices == new_slices)
3626 return;
3627
3628 WARN_ON(!new_dbuf_state->base.changed);
3629
3630 gen9_dbuf_slices_update(display, req_slices: new_slices);
3631}
3632
3633void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3634{
3635 struct intel_display *display = to_intel_display(state);
3636 const struct intel_dbuf_state *new_dbuf_state =
3637 intel_atomic_get_new_dbuf_state(state);
3638 const struct intel_dbuf_state *old_dbuf_state =
3639 intel_atomic_get_old_dbuf_state(state);
3640 u8 old_slices, new_slices;
3641
3642 if (!new_dbuf_state)
3643 return;
3644
3645 old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3646 new_slices = new_dbuf_state->enabled_slices;
3647
3648 if (old_slices == new_slices)
3649 return;
3650
3651 WARN_ON(!new_dbuf_state->base.changed);
3652
3653 gen9_dbuf_slices_update(display, req_slices: new_slices);
3654}
3655
3656int intel_dbuf_num_enabled_slices(const struct intel_dbuf_state *dbuf_state)
3657{
3658 return hweight8(dbuf_state->enabled_slices);
3659}
3660
3661int intel_dbuf_num_active_pipes(const struct intel_dbuf_state *dbuf_state)
3662{
3663 return hweight8(dbuf_state->active_pipes);
3664}
3665
3666bool intel_dbuf_pmdemand_needs_update(struct intel_atomic_state *state)
3667{
3668 struct intel_display *display = to_intel_display(state);
3669 const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3670
3671 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3672 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3673
3674 if (new_dbuf_state &&
3675 new_dbuf_state->active_pipes != old_dbuf_state->active_pipes)
3676 return true;
3677
3678 if (DISPLAY_VER(display) < 30) {
3679 if (new_dbuf_state &&
3680 new_dbuf_state->enabled_slices !=
3681 old_dbuf_state->enabled_slices)
3682 return true;
3683 }
3684
3685 return false;
3686}
3687
3688static void skl_mbus_sanitize(struct intel_display *display)
3689{
3690 struct intel_dbuf_state *dbuf_state =
3691 to_intel_dbuf_state(display->dbuf.obj.state);
3692
3693 if (!HAS_MBUS_JOINING(display))
3694 return;
3695
3696 if (!dbuf_state->joined_mbus ||
3697 adlp_check_mbus_joined(active_pipes: dbuf_state->active_pipes))
3698 return;
3699
3700 drm_dbg_kms(display->drm, "Disabling redundant MBUS joining (active pipes 0x%x)\n",
3701 dbuf_state->active_pipes);
3702
3703 dbuf_state->joined_mbus = false;
3704 intel_dbuf_mdclk_cdclk_ratio_update(display,
3705 ratio: dbuf_state->mdclk_cdclk_ratio,
3706 joined_mbus: dbuf_state->joined_mbus);
3707 pipe_mbus_dbox_ctl_update(display, dbuf_state);
3708 mbus_ctl_join_update(display, dbuf_state, pipe: INVALID_PIPE);
3709}
3710
3711static bool skl_dbuf_is_misconfigured(struct intel_display *display)
3712{
3713 const struct intel_dbuf_state *dbuf_state =
3714 to_intel_dbuf_state(display->dbuf.obj.state);
3715 struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3716 struct intel_crtc *crtc;
3717
3718 for_each_intel_crtc(display->drm, crtc) {
3719 const struct intel_crtc_state *crtc_state =
3720 to_intel_crtc_state(crtc->base.state);
3721
3722 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3723 }
3724
3725 for_each_intel_crtc(display->drm, crtc) {
3726 const struct intel_crtc_state *crtc_state =
3727 to_intel_crtc_state(crtc->base.state);
3728 u8 slices;
3729
3730 slices = skl_compute_dbuf_slices(crtc, active_pipes: dbuf_state->active_pipes,
3731 join_mbus: dbuf_state->joined_mbus);
3732 if (dbuf_state->slices[crtc->pipe] & ~slices)
3733 return true;
3734
3735 if (skl_ddb_allocation_overlaps(ddb: &crtc_state->wm.skl.ddb, entries,
3736 num_entries: I915_MAX_PIPES, ignore_idx: crtc->pipe))
3737 return true;
3738 }
3739
3740 return false;
3741}
3742
3743static void skl_dbuf_sanitize(struct intel_display *display)
3744{
3745 struct intel_crtc *crtc;
3746
3747 /*
3748 * On TGL/RKL (at least) the BIOS likes to assign the planes
3749 * to the wrong DBUF slices. This will cause an infinite loop
3750 * in skl_commit_modeset_enables() as it can't find a way to
3751 * transition between the old bogus DBUF layout to the new
3752 * proper DBUF layout without DBUF allocation overlaps between
3753 * the planes (which cannot be allowed or else the hardware
3754 * may hang). If we detect a bogus DBUF layout just turn off
3755 * all the planes so that skl_commit_modeset_enables() can
3756 * simply ignore them.
3757 */
3758 if (!skl_dbuf_is_misconfigured(display))
3759 return;
3760
3761 drm_dbg_kms(display->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3762
3763 for_each_intel_crtc(display->drm, crtc) {
3764 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3765 const struct intel_plane_state *plane_state =
3766 to_intel_plane_state(plane->base.state);
3767 struct intel_crtc_state *crtc_state =
3768 to_intel_crtc_state(crtc->base.state);
3769
3770 if (plane_state->uapi.visible)
3771 intel_plane_disable_noatomic(crtc, plane);
3772
3773 drm_WARN_ON(display->drm, crtc_state->active_planes != 0);
3774
3775 memset(s: &crtc_state->wm.skl.ddb, c: 0, n: sizeof(crtc_state->wm.skl.ddb));
3776 }
3777}
3778
3779static void skl_wm_sanitize(struct intel_display *display)
3780{
3781 skl_mbus_sanitize(display);
3782 skl_dbuf_sanitize(display);
3783}
3784
3785void skl_wm_crtc_disable_noatomic(struct intel_crtc *crtc)
3786{
3787 struct intel_display *display = to_intel_display(crtc);
3788 struct intel_crtc_state *crtc_state =
3789 to_intel_crtc_state(crtc->base.state);
3790 struct intel_dbuf_state *dbuf_state =
3791 to_intel_dbuf_state(display->dbuf.obj.state);
3792 enum pipe pipe = crtc->pipe;
3793
3794 if (DISPLAY_VER(display) < 9)
3795 return;
3796
3797 dbuf_state->active_pipes &= ~BIT(pipe);
3798
3799 dbuf_state->weight[pipe] = 0;
3800 dbuf_state->slices[pipe] = 0;
3801
3802 memset(s: &dbuf_state->ddb[pipe], c: 0, n: sizeof(dbuf_state->ddb[pipe]));
3803
3804 memset(s: &crtc_state->wm.skl.ddb, c: 0, n: sizeof(crtc_state->wm.skl.ddb));
3805}
3806
3807void skl_wm_plane_disable_noatomic(struct intel_crtc *crtc,
3808 struct intel_plane *plane)
3809{
3810 struct intel_display *display = to_intel_display(crtc);
3811 struct intel_crtc_state *crtc_state =
3812 to_intel_crtc_state(crtc->base.state);
3813
3814 if (DISPLAY_VER(display) < 9)
3815 return;
3816
3817 skl_ddb_entry_init(entry: &crtc_state->wm.skl.plane_ddb[plane->id], start: 0, end: 0);
3818 skl_ddb_entry_init(entry: &crtc_state->wm.skl.plane_ddb[plane->id], start: 0, end: 0);
3819
3820 crtc_state->wm.skl.plane_min_ddb[plane->id] = 0;
3821 crtc_state->wm.skl.plane_interim_ddb[plane->id] = 0;
3822
3823 memset(s: &crtc_state->wm.skl.raw.planes[plane->id], c: 0,
3824 n: sizeof(crtc_state->wm.skl.raw.planes[plane->id]));
3825 memset(s: &crtc_state->wm.skl.optimal.planes[plane->id], c: 0,
3826 n: sizeof(crtc_state->wm.skl.optimal.planes[plane->id]));
3827}
3828
3829void intel_wm_state_verify(struct intel_atomic_state *state,
3830 struct intel_crtc *crtc)
3831{
3832 struct intel_display *display = to_intel_display(state);
3833 const struct intel_crtc_state *new_crtc_state =
3834 intel_atomic_get_new_crtc_state(state, crtc);
3835 struct skl_hw_state {
3836 struct skl_ddb_entry ddb[I915_MAX_PLANES];
3837 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3838 u16 min_ddb[I915_MAX_PLANES];
3839 u16 interim_ddb[I915_MAX_PLANES];
3840 struct skl_pipe_wm wm;
3841 } *hw;
3842 const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3843 struct intel_plane *plane;
3844 u8 hw_enabled_slices;
3845 int level;
3846
3847 if (DISPLAY_VER(display) < 9 || !new_crtc_state->hw.active)
3848 return;
3849
3850 hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3851 if (!hw)
3852 return;
3853
3854 skl_pipe_wm_get_hw_state(crtc, out: &hw->wm);
3855
3856 skl_pipe_ddb_get_hw_state(crtc, ddb: hw->ddb, ddb_y: hw->ddb_y, min_ddb: hw->min_ddb, interim_ddb: hw->interim_ddb);
3857
3858 hw_enabled_slices = intel_enabled_dbuf_slices_mask(display);
3859
3860 if (DISPLAY_VER(display) >= 11 &&
3861 hw_enabled_slices != display->dbuf.enabled_slices)
3862 drm_err(display->drm,
3863 "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3864 display->dbuf.enabled_slices,
3865 hw_enabled_slices);
3866
3867 for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
3868 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3869 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3870
3871 /* Watermarks */
3872 for (level = 0; level < display->wm.num_levels; level++) {
3873 hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3874 sw_wm_level = skl_plane_wm_level(pipe_wm: sw_wm, plane_id: plane->id, level);
3875
3876 if (skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level))
3877 continue;
3878
3879 drm_err(display->drm,
3880 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3881 plane->base.base.id, plane->base.name, level,
3882 sw_wm_level->enable,
3883 sw_wm_level->blocks,
3884 sw_wm_level->lines,
3885 hw_wm_level->enable,
3886 hw_wm_level->blocks,
3887 hw_wm_level->lines);
3888 }
3889
3890 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3891 sw_wm_level = skl_plane_trans_wm(pipe_wm: sw_wm, plane_id: plane->id);
3892
3893 if (!skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3894 drm_err(display->drm,
3895 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3896 plane->base.base.id, plane->base.name,
3897 sw_wm_level->enable,
3898 sw_wm_level->blocks,
3899 sw_wm_level->lines,
3900 hw_wm_level->enable,
3901 hw_wm_level->blocks,
3902 hw_wm_level->lines);
3903 }
3904
3905 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3906 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3907
3908 if (HAS_HW_SAGV_WM(display) &&
3909 !skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3910 drm_err(display->drm,
3911 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3912 plane->base.base.id, plane->base.name,
3913 sw_wm_level->enable,
3914 sw_wm_level->blocks,
3915 sw_wm_level->lines,
3916 hw_wm_level->enable,
3917 hw_wm_level->blocks,
3918 hw_wm_level->lines);
3919 }
3920
3921 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3922 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3923
3924 if (HAS_HW_SAGV_WM(display) &&
3925 !skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3926 drm_err(display->drm,
3927 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3928 plane->base.base.id, plane->base.name,
3929 sw_wm_level->enable,
3930 sw_wm_level->blocks,
3931 sw_wm_level->lines,
3932 hw_wm_level->enable,
3933 hw_wm_level->blocks,
3934 hw_wm_level->lines);
3935 }
3936
3937 /* DDB */
3938 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3939 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3940
3941 if (!skl_ddb_entry_equal(e1: hw_ddb_entry, e2: sw_ddb_entry)) {
3942 drm_err(display->drm,
3943 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3944 plane->base.base.id, plane->base.name,
3945 sw_ddb_entry->start, sw_ddb_entry->end,
3946 hw_ddb_entry->start, hw_ddb_entry->end);
3947 }
3948 }
3949
3950 kfree(objp: hw);
3951}
3952
3953static const struct intel_wm_funcs skl_wm_funcs = {
3954 .compute_global_watermarks = skl_compute_wm,
3955 .get_hw_state = skl_wm_get_hw_state,
3956 .sanitize = skl_wm_sanitize,
3957};
3958
3959void skl_wm_init(struct intel_display *display)
3960{
3961 intel_sagv_init(display);
3962
3963 skl_setup_wm_latency(display);
3964
3965 display->funcs.wm = &skl_wm_funcs;
3966}
3967
3968static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3969{
3970 struct intel_display *display = m->private;
3971
3972 seq_printf(m, fmt: "Isochronous Priority Control: %s\n",
3973 str_yes_no(v: skl_watermark_ipc_enabled(display)));
3974 return 0;
3975}
3976
3977static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3978{
3979 struct intel_display *display = inode->i_private;
3980
3981 return single_open(file, skl_watermark_ipc_status_show, display);
3982}
3983
3984static ssize_t skl_watermark_ipc_status_write(struct file *file,
3985 const char __user *ubuf,
3986 size_t len, loff_t *offp)
3987{
3988 struct seq_file *m = file->private_data;
3989 struct intel_display *display = m->private;
3990 bool enable;
3991 int ret;
3992
3993 ret = kstrtobool_from_user(s: ubuf, count: len, res: &enable);
3994 if (ret < 0)
3995 return ret;
3996
3997 with_intel_display_rpm(display) {
3998 if (!skl_watermark_ipc_enabled(display) && enable)
3999 drm_info(display->drm,
4000 "Enabling IPC: WM will be proper only after next commit\n");
4001 display->wm.ipc_enabled = enable;
4002 skl_watermark_ipc_update(display);
4003 }
4004
4005 return len;
4006}
4007
4008static const struct file_operations skl_watermark_ipc_status_fops = {
4009 .owner = THIS_MODULE,
4010 .open = skl_watermark_ipc_status_open,
4011 .read = seq_read,
4012 .llseek = seq_lseek,
4013 .release = single_release,
4014 .write = skl_watermark_ipc_status_write
4015};
4016
4017static int intel_sagv_status_show(struct seq_file *m, void *unused)
4018{
4019 struct intel_display *display = m->private;
4020 static const char * const sagv_status[] = {
4021 [I915_SAGV_UNKNOWN] = "unknown",
4022 [I915_SAGV_DISABLED] = "disabled",
4023 [I915_SAGV_ENABLED] = "enabled",
4024 [I915_SAGV_NOT_CONTROLLED] = "not controlled",
4025 };
4026
4027 seq_printf(m, fmt: "SAGV available: %s\n", str_yes_no(v: intel_has_sagv(display)));
4028 seq_printf(m, fmt: "SAGV modparam: %s\n",
4029 str_enabled_disabled(v: display->params.enable_sagv));
4030 seq_printf(m, fmt: "SAGV status: %s\n", sagv_status[display->sagv.status]);
4031 seq_printf(m, fmt: "SAGV block time: %d usec\n", display->sagv.block_time_us);
4032
4033 return 0;
4034}
4035
4036DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
4037
4038void skl_watermark_debugfs_register(struct intel_display *display)
4039{
4040 struct dentry *debugfs_root = display->drm->debugfs_root;
4041
4042 if (HAS_IPC(display))
4043 debugfs_create_file("i915_ipc_status", 0644, debugfs_root,
4044 display, &skl_watermark_ipc_status_fops);
4045
4046 if (HAS_SAGV(display))
4047 debugfs_create_file("i915_sagv_status", 0444, debugfs_root,
4048 display, &intel_sagv_status_fops);
4049}
4050
4051unsigned int skl_watermark_max_latency(struct intel_display *display, int initial_wm_level)
4052{
4053 int level;
4054
4055 for (level = display->wm.num_levels - 1; level >= initial_wm_level; level--) {
4056 unsigned int latency = skl_wm_latency(display, level, NULL);
4057
4058 if (latency)
4059 return latency;
4060 }
4061
4062 return 0;
4063}
4064