1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023 Intel Corporation
4 */
5
6#include <linux/iopoll.h>
7
8#include "soc/intel_dram.h"
9
10#include "i915_drv.h"
11#include "i915_reg.h"
12#include "i9xx_wm.h"
13#include "i9xx_wm_regs.h"
14#include "intel_atomic.h"
15#include "intel_bo.h"
16#include "intel_de.h"
17#include "intel_display.h"
18#include "intel_display_regs.h"
19#include "intel_display_trace.h"
20#include "intel_fb.h"
21#include "intel_mchbar_regs.h"
22#include "intel_wm.h"
23#include "skl_watermark.h"
24#include "vlv_sideband.h"
25
26struct intel_watermark_params {
27 u16 fifo_size;
28 u16 max_wm;
29 u8 default_wm;
30 u8 guard_size;
31 u8 cacheline_size;
32};
33
34/* used in computing the new watermarks state */
35struct intel_wm_config {
36 unsigned int num_pipes_active;
37 bool sprites_enabled;
38 bool sprites_scaled;
39};
40
41struct cxsr_latency {
42 bool is_desktop : 1;
43 bool is_ddr3 : 1;
44 u16 fsb_freq;
45 u16 mem_freq;
46 u16 display_sr;
47 u16 display_hpll_disable;
48 u16 cursor_sr;
49 u16 cursor_hpll_disable;
50};
51
52static const struct cxsr_latency cxsr_latency_table[] = {
53 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
54 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
55 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
56 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
57 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
58
59 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
60 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
61 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
62 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
63 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
64
65 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
66 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
67 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
68 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
69 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
70
71 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
72 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
73 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
74 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
75 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
76
77 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
78 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
79 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
80 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
81 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
82
83 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
84 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
85 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
86 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
87 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
88};
89
90static const struct cxsr_latency *pnv_get_cxsr_latency(struct intel_display *display)
91{
92 const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
93 bool is_ddr3 = dram_info->type == INTEL_DRAM_DDR3;
94 int i;
95
96 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
97 const struct cxsr_latency *latency = &cxsr_latency_table[i];
98 bool is_desktop = !display->platform.mobile;
99
100 if (is_desktop == latency->is_desktop &&
101 is_ddr3 == latency->is_ddr3 &&
102 DIV_ROUND_CLOSEST(dram_info->fsb_freq, 1000) == latency->fsb_freq &&
103 DIV_ROUND_CLOSEST(dram_info->mem_freq, 1000) == latency->mem_freq)
104 return latency;
105 }
106
107 drm_dbg_kms(display->drm,
108 "Could not find CxSR latency for %s, FSB %u kHz, MEM %u kHz\n",
109 intel_dram_type_str(dram_info->type),
110 dram_info->fsb_freq, dram_info->mem_freq);
111
112 return NULL;
113}
114
115static void chv_set_memory_dvfs(struct intel_display *display, bool enable)
116{
117 u32 val;
118 int ret;
119
120 vlv_punit_get(drm: display->drm);
121
122 val = vlv_punit_read(drm: display->drm, PUNIT_REG_DDR_SETUP2);
123 if (enable)
124 val &= ~FORCE_DDR_HIGH_FREQ;
125 else
126 val |= FORCE_DDR_HIGH_FREQ;
127 val &= ~FORCE_DDR_LOW_FREQ;
128 val |= FORCE_DDR_FREQ_REQ_ACK;
129 vlv_punit_write(drm: display->drm, PUNIT_REG_DDR_SETUP2, val);
130
131 ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
132 (val & FORCE_DDR_FREQ_REQ_ACK) == 0,
133 500, 3000, false);
134 if (ret)
135 drm_err(display->drm,
136 "timed out waiting for Punit DDR DVFS request\n");
137
138 vlv_punit_put(drm: display->drm);
139}
140
141static void chv_set_memory_pm5(struct intel_display *display, bool enable)
142{
143 u32 val;
144
145 vlv_punit_get(drm: display->drm);
146
147 val = vlv_punit_read(drm: display->drm, PUNIT_REG_DSPSSPM);
148 if (enable)
149 val |= DSP_MAXFIFO_PM5_ENABLE;
150 else
151 val &= ~DSP_MAXFIFO_PM5_ENABLE;
152 vlv_punit_write(drm: display->drm, PUNIT_REG_DSPSSPM, val);
153
154 vlv_punit_put(drm: display->drm);
155}
156
157#define FW_WM(value, plane) \
158 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
159
160static bool _intel_set_memory_cxsr(struct intel_display *display, bool enable)
161{
162 bool was_enabled;
163 u32 val;
164
165 if (display->platform.valleyview || display->platform.cherryview) {
166 was_enabled = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
167 intel_de_write(display, FW_BLC_SELF_VLV, val: enable ? FW_CSPWRDWNEN : 0);
168 intel_de_posting_read(display, FW_BLC_SELF_VLV);
169 } else if (display->platform.g4x || display->platform.i965gm) {
170 was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
171 intel_de_write(display, FW_BLC_SELF, val: enable ? FW_BLC_SELF_EN : 0);
172 intel_de_posting_read(display, FW_BLC_SELF);
173 } else if (display->platform.pineview) {
174 val = intel_de_read(display, DSPFW3(display));
175 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
176 if (enable)
177 val |= PINEVIEW_SELF_REFRESH_EN;
178 else
179 val &= ~PINEVIEW_SELF_REFRESH_EN;
180 intel_de_write(display, DSPFW3(display), val);
181 intel_de_posting_read(display, DSPFW3(display));
182 } else if (display->platform.i945g || display->platform.i945gm) {
183 was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
184 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
185 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
186 intel_de_write(display, FW_BLC_SELF, val);
187 intel_de_posting_read(display, FW_BLC_SELF);
188 } else if (display->platform.i915gm) {
189 /*
190 * FIXME can't find a bit like this for 915G, and
191 * yet it does have the related watermark in
192 * FW_BLC_SELF. What's going on?
193 */
194 was_enabled = intel_de_read(display, INSTPM) & INSTPM_SELF_EN;
195 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
196 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
197 intel_de_write(display, INSTPM, val);
198 intel_de_posting_read(display, INSTPM);
199 } else {
200 return false;
201 }
202
203 trace_intel_memory_cxsr(display, old: was_enabled, new: enable);
204
205 drm_dbg_kms(display->drm, "memory self-refresh is %s (was %s)\n",
206 str_enabled_disabled(enable),
207 str_enabled_disabled(was_enabled));
208
209 return was_enabled;
210}
211
212/**
213 * intel_set_memory_cxsr - Configure CxSR state
214 * @display: display device
215 * @enable: Allow vs. disallow CxSR
216 *
217 * Allow or disallow the system to enter a special CxSR
218 * (C-state self refresh) state. What typically happens in CxSR mode
219 * is that several display FIFOs may get combined into a single larger
220 * FIFO for a particular plane (so called max FIFO mode) to allow the
221 * system to defer memory fetches longer, and the memory will enter
222 * self refresh.
223 *
224 * Note that enabling CxSR does not guarantee that the system enter
225 * this special mode, nor does it guarantee that the system stays
226 * in that mode once entered. So this just allows/disallows the system
227 * to autonomously utilize the CxSR mode. Other factors such as core
228 * C-states will affect when/if the system actually enters/exits the
229 * CxSR mode.
230 *
231 * Note that on VLV/CHV this actually only controls the max FIFO mode,
232 * and the system is free to enter/exit memory self refresh at any time
233 * even when the use of CxSR has been disallowed.
234 *
235 * While the system is actually in the CxSR/max FIFO mode, some plane
236 * control registers will not get latched on vblank. Thus in order to
237 * guarantee the system will respond to changes in the plane registers
238 * we must always disallow CxSR prior to making changes to those registers.
239 * Unfortunately the system will re-evaluate the CxSR conditions at
240 * frame start which happens after vblank start (which is when the plane
241 * registers would get latched), so we can't proceed with the plane update
242 * during the same frame where we disallowed CxSR.
243 *
244 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
245 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
246 * the hardware w.r.t. HPLL SR when writing to plane registers.
247 * Disallowing just CxSR is sufficient.
248 */
249bool intel_set_memory_cxsr(struct intel_display *display, bool enable)
250{
251 bool ret;
252
253 mutex_lock(lock: &display->wm.wm_mutex);
254 ret = _intel_set_memory_cxsr(display, enable);
255 if (display->platform.valleyview || display->platform.cherryview)
256 display->wm.vlv.cxsr = enable;
257 else if (display->platform.g4x)
258 display->wm.g4x.cxsr = enable;
259 mutex_unlock(lock: &display->wm.wm_mutex);
260
261 return ret;
262}
263
264/*
265 * Latency for FIFO fetches is dependent on several factors:
266 * - memory configuration (speed, channels)
267 * - chipset
268 * - current MCH state
269 * It can be fairly high in some situations, so here we assume a fairly
270 * pessimal value. It's a tradeoff between extra memory fetches (if we
271 * set this value too high, the FIFO will fetch frequently to stay full)
272 * and power consumption (set it too low to save power and we might see
273 * FIFO underruns and display "flicker").
274 *
275 * A value of 5us seems to be a good balance; safe for very low end
276 * platforms but not overly aggressive on lower latency configs.
277 */
278static const int pessimal_latency_ns = 5000;
279
280#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
281 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
282
283static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
284{
285 struct intel_display *display = to_intel_display(crtc_state);
286 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
287 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
288 enum pipe pipe = crtc->pipe;
289 int sprite0_start, sprite1_start;
290 u32 dsparb, dsparb2, dsparb3;
291
292 switch (pipe) {
293 case PIPE_A:
294 dsparb = intel_de_read(display, DSPARB(display));
295 dsparb2 = intel_de_read(display, DSPARB2);
296 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
297 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
298 break;
299 case PIPE_B:
300 dsparb = intel_de_read(display, DSPARB(display));
301 dsparb2 = intel_de_read(display, DSPARB2);
302 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
303 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
304 break;
305 case PIPE_C:
306 dsparb2 = intel_de_read(display, DSPARB2);
307 dsparb3 = intel_de_read(display, DSPARB3);
308 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
309 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
310 break;
311 default:
312 MISSING_CASE(pipe);
313 return;
314 }
315
316 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
317 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
318 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
319 fifo_state->plane[PLANE_CURSOR] = 63;
320}
321
322static int i9xx_get_fifo_size(struct intel_display *display,
323 enum i9xx_plane_id i9xx_plane)
324{
325 u32 dsparb = intel_de_read(display, DSPARB(display));
326 int size;
327
328 size = dsparb & 0x7f;
329 if (i9xx_plane == PLANE_B)
330 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
331
332 drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
333 dsparb, plane_name(i9xx_plane), size);
334
335 return size;
336}
337
338static int i830_get_fifo_size(struct intel_display *display,
339 enum i9xx_plane_id i9xx_plane)
340{
341 u32 dsparb = intel_de_read(display, DSPARB(display));
342 int size;
343
344 size = dsparb & 0x1ff;
345 if (i9xx_plane == PLANE_B)
346 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
347 size >>= 1; /* Convert to cachelines */
348
349 drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
350 dsparb, plane_name(i9xx_plane), size);
351
352 return size;
353}
354
355static int i845_get_fifo_size(struct intel_display *display,
356 enum i9xx_plane_id i9xx_plane)
357{
358 u32 dsparb = intel_de_read(display, DSPARB(display));
359 int size;
360
361 size = dsparb & 0x7f;
362 size >>= 2; /* Convert to cachelines */
363
364 drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
365 dsparb, plane_name(i9xx_plane), size);
366
367 return size;
368}
369
370/* Pineview has different values for various configs */
371static const struct intel_watermark_params pnv_display_wm = {
372 .fifo_size = PINEVIEW_DISPLAY_FIFO,
373 .max_wm = PINEVIEW_MAX_WM,
374 .default_wm = PINEVIEW_DFT_WM,
375 .guard_size = PINEVIEW_GUARD_WM,
376 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
377};
378
379static const struct intel_watermark_params pnv_display_hplloff_wm = {
380 .fifo_size = PINEVIEW_DISPLAY_FIFO,
381 .max_wm = PINEVIEW_MAX_WM,
382 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
383 .guard_size = PINEVIEW_GUARD_WM,
384 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
385};
386
387static const struct intel_watermark_params pnv_cursor_wm = {
388 .fifo_size = PINEVIEW_CURSOR_FIFO,
389 .max_wm = PINEVIEW_CURSOR_MAX_WM,
390 .default_wm = PINEVIEW_CURSOR_DFT_WM,
391 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
392 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
393};
394
395static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
396 .fifo_size = PINEVIEW_CURSOR_FIFO,
397 .max_wm = PINEVIEW_CURSOR_MAX_WM,
398 .default_wm = PINEVIEW_CURSOR_DFT_WM,
399 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
400 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
401};
402
403static const struct intel_watermark_params i965_cursor_wm_info = {
404 .fifo_size = I965_CURSOR_FIFO,
405 .max_wm = I965_CURSOR_MAX_WM,
406 .default_wm = I965_CURSOR_DFT_WM,
407 .guard_size = 2,
408 .cacheline_size = I915_FIFO_LINE_SIZE,
409};
410
411static const struct intel_watermark_params i945_wm_info = {
412 .fifo_size = I945_FIFO_SIZE,
413 .max_wm = I915_MAX_WM,
414 .default_wm = 1,
415 .guard_size = 2,
416 .cacheline_size = I915_FIFO_LINE_SIZE,
417};
418
419static const struct intel_watermark_params i915_wm_info = {
420 .fifo_size = I915_FIFO_SIZE,
421 .max_wm = I915_MAX_WM,
422 .default_wm = 1,
423 .guard_size = 2,
424 .cacheline_size = I915_FIFO_LINE_SIZE,
425};
426
427static const struct intel_watermark_params i830_a_wm_info = {
428 .fifo_size = I855GM_FIFO_SIZE,
429 .max_wm = I915_MAX_WM,
430 .default_wm = 1,
431 .guard_size = 2,
432 .cacheline_size = I830_FIFO_LINE_SIZE,
433};
434
435static const struct intel_watermark_params i830_bc_wm_info = {
436 .fifo_size = I855GM_FIFO_SIZE,
437 .max_wm = I915_MAX_WM / 2,
438 .default_wm = 1,
439 .guard_size = 2,
440 .cacheline_size = I830_FIFO_LINE_SIZE,
441};
442
443static const struct intel_watermark_params i845_wm_info = {
444 .fifo_size = I830_FIFO_SIZE,
445 .max_wm = I915_MAX_WM,
446 .default_wm = 1,
447 .guard_size = 2,
448 .cacheline_size = I830_FIFO_LINE_SIZE,
449};
450
451/**
452 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
453 * @pixel_rate: Pipe pixel rate in kHz
454 * @cpp: Plane bytes per pixel
455 * @latency: Memory wakeup latency in 0.1us units
456 *
457 * Compute the watermark using the method 1 or "small buffer"
458 * formula. The caller may additionally add extra cachelines
459 * to account for TLB misses and clock crossings.
460 *
461 * This method is concerned with the short term drain rate
462 * of the FIFO, ie. it does not account for blanking periods
463 * which would effectively reduce the average drain rate across
464 * a longer period. The name "small" refers to the fact the
465 * FIFO is relatively small compared to the amount of data
466 * fetched.
467 *
468 * The FIFO level vs. time graph might look something like:
469 *
470 * |\ |\
471 * | \ | \
472 * __---__---__ (- plane active, _ blanking)
473 * -> time
474 *
475 * or perhaps like this:
476 *
477 * |\|\ |\|\
478 * __----__----__ (- plane active, _ blanking)
479 * -> time
480 *
481 * Returns:
482 * The watermark in bytes
483 */
484static unsigned int intel_wm_method1(unsigned int pixel_rate,
485 unsigned int cpp,
486 unsigned int latency)
487{
488 u64 ret;
489
490 ret = mul_u32_u32(a: pixel_rate, b: cpp * latency);
491 ret = DIV_ROUND_UP_ULL(ret, 10000);
492
493 return ret;
494}
495
496/**
497 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
498 * @pixel_rate: Pipe pixel rate in kHz
499 * @htotal: Pipe horizontal total
500 * @width: Plane width in pixels
501 * @cpp: Plane bytes per pixel
502 * @latency: Memory wakeup latency in 0.1us units
503 *
504 * Compute the watermark using the method 2 or "large buffer"
505 * formula. The caller may additionally add extra cachelines
506 * to account for TLB misses and clock crossings.
507 *
508 * This method is concerned with the long term drain rate
509 * of the FIFO, ie. it does account for blanking periods
510 * which effectively reduce the average drain rate across
511 * a longer period. The name "large" refers to the fact the
512 * FIFO is relatively large compared to the amount of data
513 * fetched.
514 *
515 * The FIFO level vs. time graph might look something like:
516 *
517 * |\___ |\___
518 * | \___ | \___
519 * | \ | \
520 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
521 * -> time
522 *
523 * Returns:
524 * The watermark in bytes
525 */
526static unsigned int intel_wm_method2(unsigned int pixel_rate,
527 unsigned int htotal,
528 unsigned int width,
529 unsigned int cpp,
530 unsigned int latency)
531{
532 unsigned int ret;
533
534 /*
535 * FIXME remove once all users are computing
536 * watermarks in the correct place.
537 */
538 if (WARN_ON_ONCE(htotal == 0))
539 htotal = 1;
540
541 ret = (latency * pixel_rate) / (htotal * 10000);
542 ret = (ret + 1) * width * cpp;
543
544 return ret;
545}
546
547/**
548 * intel_calculate_wm - calculate watermark level
549 * @display: display device
550 * @pixel_rate: pixel clock
551 * @wm: chip FIFO params
552 * @fifo_size: size of the FIFO buffer
553 * @cpp: bytes per pixel
554 * @latency_ns: memory latency for the platform
555 *
556 * Calculate the watermark level (the level at which the display plane will
557 * start fetching from memory again). Each chip has a different display
558 * FIFO size and allocation, so the caller needs to figure that out and pass
559 * in the correct intel_watermark_params structure.
560 *
561 * As the pixel clock runs, the FIFO will be drained at a rate that depends
562 * on the pixel size. When it reaches the watermark level, it'll start
563 * fetching FIFO line sized based chunks from memory until the FIFO fills
564 * past the watermark point. If the FIFO drains completely, a FIFO underrun
565 * will occur, and a display engine hang could result.
566 */
567static unsigned int intel_calculate_wm(struct intel_display *display,
568 int pixel_rate,
569 const struct intel_watermark_params *wm,
570 int fifo_size, int cpp,
571 unsigned int latency_ns)
572{
573 int entries, wm_size;
574
575 /*
576 * Note: we need to make sure we don't overflow for various clock &
577 * latency values.
578 * clocks go from a few thousand to several hundred thousand.
579 * latency is usually a few thousand
580 */
581 entries = intel_wm_method1(pixel_rate, cpp,
582 latency: latency_ns / 100);
583 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
584 wm->guard_size;
585 drm_dbg_kms(display->drm, "FIFO entries required for mode: %d\n", entries);
586
587 wm_size = fifo_size - entries;
588 drm_dbg_kms(display->drm, "FIFO watermark level: %d\n", wm_size);
589
590 /* Don't promote wm_size to unsigned... */
591 if (wm_size > wm->max_wm)
592 wm_size = wm->max_wm;
593 if (wm_size <= 0)
594 wm_size = wm->default_wm;
595
596 /*
597 * Bspec seems to indicate that the value shouldn't be lower than
598 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
599 * Lets go for 8 which is the burst size since certain platforms
600 * already use a hardcoded 8 (which is what the spec says should be
601 * done).
602 */
603 if (wm_size <= 8)
604 wm_size = 8;
605
606 return wm_size;
607}
608
609static bool is_disabling(int old, int new, int threshold)
610{
611 return old >= threshold && new < threshold;
612}
613
614static bool is_enabling(int old, int new, int threshold)
615{
616 return old < threshold && new >= threshold;
617}
618
619static bool intel_crtc_active(struct intel_crtc *crtc)
620{
621 /* Be paranoid as we can arrive here with only partial
622 * state retrieved from the hardware during setup.
623 *
624 * We can ditch the adjusted_mode.crtc_clock check as soon
625 * as Haswell has gained clock readout/fastboot support.
626 *
627 * We can ditch the crtc->primary->state->fb check as soon as we can
628 * properly reconstruct framebuffers.
629 *
630 * FIXME: The intel_crtc->active here should be switched to
631 * crtc->state->active once we have proper CRTC states wired up
632 * for atomic.
633 */
634 return crtc->active && crtc->base.primary->state->fb &&
635 crtc->config->hw.adjusted_mode.crtc_clock;
636}
637
638static struct intel_crtc *single_enabled_crtc(struct intel_display *display)
639{
640 struct intel_crtc *crtc, *enabled = NULL;
641
642 for_each_intel_crtc(display->drm, crtc) {
643 if (intel_crtc_active(crtc)) {
644 if (enabled)
645 return NULL;
646 enabled = crtc;
647 }
648 }
649
650 return enabled;
651}
652
653static void pnv_update_wm(struct intel_display *display)
654{
655 struct intel_crtc *crtc;
656 const struct cxsr_latency *latency;
657 u32 reg;
658 unsigned int wm;
659
660 latency = pnv_get_cxsr_latency(display);
661 if (!latency) {
662 drm_dbg_kms(display->drm, "Unknown FSB/MEM, disabling CxSR\n");
663 intel_set_memory_cxsr(display, enable: false);
664 return;
665 }
666
667 crtc = single_enabled_crtc(display);
668 if (crtc) {
669 const struct drm_framebuffer *fb =
670 crtc->base.primary->state->fb;
671 int pixel_rate = crtc->config->pixel_rate;
672 int cpp = fb->format->cpp[0];
673
674 /* Display SR */
675 wm = intel_calculate_wm(display, pixel_rate,
676 wm: &pnv_display_wm,
677 fifo_size: pnv_display_wm.fifo_size,
678 cpp, latency_ns: latency->display_sr);
679 reg = intel_de_read(display, DSPFW1(display));
680 reg &= ~DSPFW_SR_MASK;
681 reg |= FW_WM(wm, SR);
682 intel_de_write(display, DSPFW1(display), val: reg);
683 drm_dbg_kms(display->drm, "DSPFW1 register is %x\n", reg);
684
685 /* cursor SR */
686 wm = intel_calculate_wm(display, pixel_rate,
687 wm: &pnv_cursor_wm,
688 fifo_size: pnv_display_wm.fifo_size,
689 cpp: 4, latency_ns: latency->cursor_sr);
690 intel_de_rmw(display, DSPFW3(display),
691 DSPFW_CURSOR_SR_MASK, FW_WM(wm, CURSOR_SR));
692
693 /* Display HPLL off SR */
694 wm = intel_calculate_wm(display, pixel_rate,
695 wm: &pnv_display_hplloff_wm,
696 fifo_size: pnv_display_hplloff_wm.fifo_size,
697 cpp, latency_ns: latency->display_hpll_disable);
698 intel_de_rmw(display, DSPFW3(display),
699 DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
700
701 /* cursor HPLL off SR */
702 wm = intel_calculate_wm(display, pixel_rate,
703 wm: &pnv_cursor_hplloff_wm,
704 fifo_size: pnv_display_hplloff_wm.fifo_size,
705 cpp: 4, latency_ns: latency->cursor_hpll_disable);
706 reg = intel_de_read(display, DSPFW3(display));
707 reg &= ~DSPFW_HPLL_CURSOR_MASK;
708 reg |= FW_WM(wm, HPLL_CURSOR);
709 intel_de_write(display, DSPFW3(display), val: reg);
710 drm_dbg_kms(display->drm, "DSPFW3 register is %x\n", reg);
711
712 intel_set_memory_cxsr(display, enable: true);
713 } else {
714 intel_set_memory_cxsr(display, enable: false);
715 }
716}
717
718static bool i9xx_wm_need_update(const struct intel_plane_state *old_plane_state,
719 const struct intel_plane_state *new_plane_state)
720{
721 /* Update watermarks on tiling or size changes. */
722 if (old_plane_state->uapi.visible != new_plane_state->uapi.visible)
723 return true;
724
725 if (!old_plane_state->hw.fb || !new_plane_state->hw.fb)
726 return false;
727
728 if (old_plane_state->hw.fb->modifier != new_plane_state->hw.fb->modifier ||
729 old_plane_state->hw.rotation != new_plane_state->hw.rotation ||
730 drm_rect_width(r: &old_plane_state->uapi.src) != drm_rect_width(r: &new_plane_state->uapi.src) ||
731 drm_rect_height(r: &old_plane_state->uapi.src) != drm_rect_height(r: &new_plane_state->uapi.src) ||
732 drm_rect_width(r: &old_plane_state->uapi.dst) != drm_rect_width(r: &new_plane_state->uapi.dst) ||
733 drm_rect_height(r: &old_plane_state->uapi.dst) != drm_rect_height(r: &new_plane_state->uapi.dst))
734 return true;
735
736 return false;
737}
738
739static void i9xx_wm_compute(struct intel_crtc_state *new_crtc_state,
740 const struct intel_plane_state *old_plane_state,
741 const struct intel_plane_state *new_plane_state)
742{
743 bool turn_off, turn_on, visible, was_visible, mode_changed;
744
745 mode_changed = intel_crtc_needs_modeset(crtc_state: new_crtc_state);
746 was_visible = old_plane_state->uapi.visible;
747 visible = new_plane_state->uapi.visible;
748
749 if (!was_visible && !visible)
750 return;
751
752 turn_off = was_visible && (!visible || mode_changed);
753 turn_on = visible && (!was_visible || mode_changed);
754
755 /* FIXME nuke when all wm code is atomic */
756 if (turn_on) {
757 new_crtc_state->update_wm_pre = true;
758 } else if (turn_off) {
759 new_crtc_state->update_wm_post = true;
760 } else if (i9xx_wm_need_update(old_plane_state, new_plane_state)) {
761 /* FIXME bollocks */
762 new_crtc_state->update_wm_pre = true;
763 new_crtc_state->update_wm_post = true;
764 }
765}
766
767static int i9xx_compute_watermarks(struct intel_atomic_state *state,
768 struct intel_crtc *crtc)
769{
770 struct intel_crtc_state *new_crtc_state =
771 intel_atomic_get_new_crtc_state(state, crtc);
772 const struct intel_plane_state *old_plane_state;
773 const struct intel_plane_state *new_plane_state;
774 struct intel_plane *plane;
775 int i;
776
777 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
778 new_plane_state, i) {
779 if (plane->pipe != crtc->pipe)
780 continue;
781
782 i9xx_wm_compute(new_crtc_state, old_plane_state, new_plane_state);
783 }
784
785 return 0;
786}
787
788/*
789 * Documentation says:
790 * "If the line size is small, the TLB fetches can get in the way of the
791 * data fetches, causing some lag in the pixel data return which is not
792 * accounted for in the above formulas. The following adjustment only
793 * needs to be applied if eight whole lines fit in the buffer at once.
794 * The WM is adjusted upwards by the difference between the FIFO size
795 * and the size of 8 whole lines. This adjustment is always performed
796 * in the actual pixel depth regardless of whether FBC is enabled or not."
797 */
798static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
799{
800 int tlb_miss = fifo_size * 64 - width * cpp * 8;
801
802 return max(0, tlb_miss);
803}
804
805static void g4x_write_wm_values(struct intel_display *display,
806 const struct g4x_wm_values *wm)
807{
808 enum pipe pipe;
809
810 for_each_pipe(display, pipe)
811 trace_g4x_wm(crtc: intel_crtc_for_pipe(display, pipe), wm);
812
813 intel_de_write(display, DSPFW1(display),
814 FW_WM(wm->sr.plane, SR) |
815 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
816 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
817 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
818 intel_de_write(display, DSPFW2(display),
819 val: (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
820 FW_WM(wm->sr.fbc, FBC_SR) |
821 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
822 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
823 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
824 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
825 intel_de_write(display, DSPFW3(display),
826 val: (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
827 FW_WM(wm->sr.cursor, CURSOR_SR) |
828 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
829 FW_WM(wm->hpll.plane, HPLL_SR));
830
831 intel_de_posting_read(display, DSPFW1(display));
832}
833
834#define FW_WM_VLV(value, plane) \
835 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
836
837static void vlv_write_wm_values(struct intel_display *display,
838 const struct vlv_wm_values *wm)
839{
840 enum pipe pipe;
841
842 for_each_pipe(display, pipe) {
843 trace_vlv_wm(crtc: intel_crtc_for_pipe(display, pipe), wm);
844
845 intel_de_write(display, VLV_DDL(pipe),
846 val: (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
847 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
848 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
849 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
850 }
851
852 /*
853 * Zero the (unused) WM1 watermarks, and also clear all the
854 * high order bits so that there are no out of bounds values
855 * present in the registers during the reprogramming.
856 */
857 intel_de_write(display, DSPHOWM, val: 0);
858 intel_de_write(display, DSPHOWM1, val: 0);
859 intel_de_write(display, DSPFW4, val: 0);
860 intel_de_write(display, DSPFW5, val: 0);
861 intel_de_write(display, DSPFW6, val: 0);
862
863 intel_de_write(display, DSPFW1(display),
864 FW_WM(wm->sr.plane, SR) |
865 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
866 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
867 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
868 intel_de_write(display, DSPFW2(display),
869 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
870 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
871 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
872 intel_de_write(display, DSPFW3(display),
873 FW_WM(wm->sr.cursor, CURSOR_SR));
874
875 if (display->platform.cherryview) {
876 intel_de_write(display, DSPFW7_CHV,
877 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
878 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
879 intel_de_write(display, DSPFW8_CHV,
880 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
881 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
882 intel_de_write(display, DSPFW9_CHV,
883 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
884 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
885 intel_de_write(display, DSPHOWM,
886 FW_WM(wm->sr.plane >> 9, SR_HI) |
887 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
888 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
889 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
890 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
891 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
892 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
893 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
894 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
895 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
896 } else {
897 intel_de_write(display, DSPFW7,
898 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
899 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
900 intel_de_write(display, DSPHOWM,
901 FW_WM(wm->sr.plane >> 9, SR_HI) |
902 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
903 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
904 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
905 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
906 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
907 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
908 }
909
910 intel_de_posting_read(display, DSPFW1(display));
911}
912
913#undef FW_WM_VLV
914
915static void g4x_setup_wm_latency(struct intel_display *display)
916{
917 /* all latencies in usec */
918 display->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
919 display->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
920 display->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
921
922 display->wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
923}
924
925static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
926{
927 /*
928 * DSPCNTR[13] supposedly controls whether the
929 * primary plane can use the FIFO space otherwise
930 * reserved for the sprite plane. It's not 100% clear
931 * what the actual FIFO size is, but it looks like we
932 * can happily set both primary and sprite watermarks
933 * up to 127 cachelines. So that would seem to mean
934 * that either DSPCNTR[13] doesn't do anything, or that
935 * the total FIFO is >= 256 cachelines in size. Either
936 * way, we don't seem to have to worry about this
937 * repartitioning as the maximum watermark value the
938 * register can hold for each plane is lower than the
939 * minimum FIFO size.
940 */
941 switch (plane_id) {
942 case PLANE_CURSOR:
943 return 63;
944 case PLANE_PRIMARY:
945 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
946 case PLANE_SPRITE0:
947 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
948 default:
949 MISSING_CASE(plane_id);
950 return 0;
951 }
952}
953
954static int g4x_fbc_fifo_size(int level)
955{
956 switch (level) {
957 case G4X_WM_LEVEL_SR:
958 return 7;
959 case G4X_WM_LEVEL_HPLL:
960 return 15;
961 default:
962 MISSING_CASE(level);
963 return 0;
964 }
965}
966
967static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
968 const struct intel_plane_state *plane_state,
969 int level)
970{
971 struct intel_display *display = to_intel_display(plane_state);
972 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
973 const struct drm_display_mode *pipe_mode =
974 &crtc_state->hw.pipe_mode;
975 unsigned int latency = display->wm.pri_latency[level] * 10;
976 unsigned int pixel_rate, htotal, cpp, width, wm;
977
978 if (latency == 0)
979 return USHRT_MAX;
980
981 if (!intel_wm_plane_visible(crtc_state, plane_state))
982 return 0;
983
984 cpp = plane_state->hw.fb->format->cpp[0];
985
986 /*
987 * WaUse32BppForSRWM:ctg,elk
988 *
989 * The spec fails to list this restriction for the
990 * HPLL watermark, which seems a little strange.
991 * Let's use 32bpp for the HPLL watermark as well.
992 */
993 if (plane->id == PLANE_PRIMARY &&
994 level != G4X_WM_LEVEL_NORMAL)
995 cpp = max(cpp, 4u);
996
997 pixel_rate = crtc_state->pixel_rate;
998 htotal = pipe_mode->crtc_htotal;
999 width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1000
1001 if (plane->id == PLANE_CURSOR) {
1002 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1003 } else if (plane->id == PLANE_PRIMARY &&
1004 level == G4X_WM_LEVEL_NORMAL) {
1005 wm = intel_wm_method1(pixel_rate, cpp, latency);
1006 } else {
1007 unsigned int small, large;
1008
1009 small = intel_wm_method1(pixel_rate, cpp, latency);
1010 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1011
1012 wm = min(small, large);
1013 }
1014
1015 wm += g4x_tlb_miss_wa(fifo_size: g4x_plane_fifo_size(plane_id: plane->id, level),
1016 width, cpp);
1017
1018 wm = DIV_ROUND_UP(wm, 64) + 2;
1019
1020 return min_t(unsigned int, wm, USHRT_MAX);
1021}
1022
1023static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1024 int level, enum plane_id plane_id, u16 value)
1025{
1026 struct intel_display *display = to_intel_display(crtc_state);
1027 bool dirty = false;
1028
1029 for (; level < display->wm.num_levels; level++) {
1030 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1031
1032 dirty |= raw->plane[plane_id] != value;
1033 raw->plane[plane_id] = value;
1034 }
1035
1036 return dirty;
1037}
1038
1039static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1040 int level, u16 value)
1041{
1042 struct intel_display *display = to_intel_display(crtc_state);
1043 bool dirty = false;
1044
1045 /* NORMAL level doesn't have an FBC watermark */
1046 level = max(level, G4X_WM_LEVEL_SR);
1047
1048 for (; level < display->wm.num_levels; level++) {
1049 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1050
1051 dirty |= raw->fbc != value;
1052 raw->fbc = value;
1053 }
1054
1055 return dirty;
1056}
1057
1058static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1059 const struct intel_plane_state *plane_state,
1060 u32 pri_val);
1061
1062static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1063 const struct intel_plane_state *plane_state)
1064{
1065 struct intel_display *display = to_intel_display(crtc_state);
1066 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1067 enum plane_id plane_id = plane->id;
1068 bool dirty = false;
1069 int level;
1070
1071 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1072 dirty |= g4x_raw_plane_wm_set(crtc_state, level: 0, plane_id, value: 0);
1073 if (plane_id == PLANE_PRIMARY)
1074 dirty |= g4x_raw_fbc_wm_set(crtc_state, level: 0, value: 0);
1075 goto out;
1076 }
1077
1078 for (level = 0; level < display->wm.num_levels; level++) {
1079 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1080 int wm, max_wm;
1081
1082 wm = g4x_compute_wm(crtc_state, plane_state, level);
1083 max_wm = g4x_plane_fifo_size(plane_id, level);
1084
1085 if (wm > max_wm)
1086 break;
1087
1088 dirty |= raw->plane[plane_id] != wm;
1089 raw->plane[plane_id] = wm;
1090
1091 if (plane_id != PLANE_PRIMARY ||
1092 level == G4X_WM_LEVEL_NORMAL)
1093 continue;
1094
1095 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1096 pri_val: raw->plane[plane_id]);
1097 max_wm = g4x_fbc_fifo_size(level);
1098
1099 /*
1100 * FBC wm is not mandatory as we
1101 * can always just disable its use.
1102 */
1103 if (wm > max_wm)
1104 wm = USHRT_MAX;
1105
1106 dirty |= raw->fbc != wm;
1107 raw->fbc = wm;
1108 }
1109
1110 /* mark watermarks as invalid */
1111 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1112
1113 if (plane_id == PLANE_PRIMARY)
1114 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1115
1116 out:
1117 if (dirty) {
1118 drm_dbg_kms(display->drm,
1119 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1120 plane->base.name,
1121 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1122 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1123 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1124
1125 if (plane_id == PLANE_PRIMARY)
1126 drm_dbg_kms(display->drm,
1127 "FBC watermarks: SR=%d, HPLL=%d\n",
1128 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1129 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1130 }
1131
1132 return dirty;
1133}
1134
1135static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1136 enum plane_id plane_id, int level)
1137{
1138 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1139
1140 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1141}
1142
1143static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1144 int level)
1145{
1146 struct intel_display *display = to_intel_display(crtc_state);
1147
1148 if (level >= display->wm.num_levels)
1149 return false;
1150
1151 return g4x_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_PRIMARY, level) &&
1152 g4x_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_SPRITE0, level) &&
1153 g4x_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_CURSOR, level);
1154}
1155
1156/* mark all levels starting from 'level' as invalid */
1157static void g4x_invalidate_wms(struct intel_crtc *crtc,
1158 struct g4x_wm_state *wm_state, int level)
1159{
1160 if (level <= G4X_WM_LEVEL_NORMAL) {
1161 enum plane_id plane_id;
1162
1163 for_each_plane_id_on_crtc(crtc, plane_id)
1164 wm_state->wm.plane[plane_id] = USHRT_MAX;
1165 }
1166
1167 if (level <= G4X_WM_LEVEL_SR) {
1168 wm_state->cxsr = false;
1169 wm_state->sr.cursor = USHRT_MAX;
1170 wm_state->sr.plane = USHRT_MAX;
1171 wm_state->sr.fbc = USHRT_MAX;
1172 }
1173
1174 if (level <= G4X_WM_LEVEL_HPLL) {
1175 wm_state->hpll_en = false;
1176 wm_state->hpll.cursor = USHRT_MAX;
1177 wm_state->hpll.plane = USHRT_MAX;
1178 wm_state->hpll.fbc = USHRT_MAX;
1179 }
1180}
1181
1182static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1183 int level)
1184{
1185 if (level < G4X_WM_LEVEL_SR)
1186 return false;
1187
1188 if (level >= G4X_WM_LEVEL_SR &&
1189 wm_state->sr.fbc > g4x_fbc_fifo_size(level: G4X_WM_LEVEL_SR))
1190 return false;
1191
1192 if (level >= G4X_WM_LEVEL_HPLL &&
1193 wm_state->hpll.fbc > g4x_fbc_fifo_size(level: G4X_WM_LEVEL_HPLL))
1194 return false;
1195
1196 return true;
1197}
1198
1199static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1200{
1201 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1202 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1203 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1204 const struct g4x_pipe_wm *raw;
1205 enum plane_id plane_id;
1206 int level;
1207
1208 level = G4X_WM_LEVEL_NORMAL;
1209 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1210 goto out;
1211
1212 raw = &crtc_state->wm.g4x.raw[level];
1213 for_each_plane_id_on_crtc(crtc, plane_id)
1214 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1215
1216 level = G4X_WM_LEVEL_SR;
1217 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1218 goto out;
1219
1220 raw = &crtc_state->wm.g4x.raw[level];
1221 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1222 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1223 wm_state->sr.fbc = raw->fbc;
1224
1225 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1226
1227 level = G4X_WM_LEVEL_HPLL;
1228 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1229 goto out;
1230
1231 raw = &crtc_state->wm.g4x.raw[level];
1232 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1233 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1234 wm_state->hpll.fbc = raw->fbc;
1235
1236 wm_state->hpll_en = wm_state->cxsr;
1237
1238 level++;
1239
1240 out:
1241 if (level == G4X_WM_LEVEL_NORMAL)
1242 return -EINVAL;
1243
1244 /* invalidate the higher levels */
1245 g4x_invalidate_wms(crtc, wm_state, level);
1246
1247 /*
1248 * Determine if the FBC watermark(s) can be used. IF
1249 * this isn't the case we prefer to disable the FBC
1250 * watermark(s) rather than disable the SR/HPLL
1251 * level(s) entirely. 'level-1' is the highest valid
1252 * level here.
1253 */
1254 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level: level - 1);
1255
1256 return 0;
1257}
1258
1259static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1260 struct intel_crtc *crtc)
1261{
1262 struct intel_crtc_state *crtc_state =
1263 intel_atomic_get_new_crtc_state(state, crtc);
1264 const struct intel_plane_state *old_plane_state;
1265 const struct intel_plane_state *new_plane_state;
1266 struct intel_plane *plane;
1267 unsigned int dirty = 0;
1268 int i;
1269
1270 for_each_oldnew_intel_plane_in_state(state, plane,
1271 old_plane_state,
1272 new_plane_state, i) {
1273 if (new_plane_state->hw.crtc != &crtc->base &&
1274 old_plane_state->hw.crtc != &crtc->base)
1275 continue;
1276
1277 if (g4x_raw_plane_wm_compute(crtc_state, plane_state: new_plane_state))
1278 dirty |= BIT(plane->id);
1279 }
1280
1281 if (!dirty)
1282 return 0;
1283
1284 return _g4x_compute_pipe_wm(crtc_state);
1285}
1286
1287static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1288 struct intel_crtc *crtc)
1289{
1290 struct intel_display *display = to_intel_display(state);
1291 struct intel_crtc_state *new_crtc_state =
1292 intel_atomic_get_new_crtc_state(state, crtc);
1293 const struct intel_crtc_state *old_crtc_state =
1294 intel_atomic_get_old_crtc_state(state, crtc);
1295 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1296 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1297 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1298 enum plane_id plane_id;
1299
1300 if (!new_crtc_state->hw.active ||
1301 intel_crtc_needs_modeset(crtc_state: new_crtc_state)) {
1302 *intermediate = *optimal;
1303
1304 intermediate->cxsr = false;
1305 intermediate->hpll_en = false;
1306 goto out;
1307 }
1308
1309 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1310 !new_crtc_state->disable_cxsr;
1311 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1312 !new_crtc_state->disable_cxsr;
1313 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1314
1315 for_each_plane_id_on_crtc(crtc, plane_id) {
1316 intermediate->wm.plane[plane_id] =
1317 max(optimal->wm.plane[plane_id],
1318 active->wm.plane[plane_id]);
1319
1320 drm_WARN_ON(display->drm, intermediate->wm.plane[plane_id] >
1321 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1322 }
1323
1324 intermediate->sr.plane = max(optimal->sr.plane,
1325 active->sr.plane);
1326 intermediate->sr.cursor = max(optimal->sr.cursor,
1327 active->sr.cursor);
1328 intermediate->sr.fbc = max(optimal->sr.fbc,
1329 active->sr.fbc);
1330
1331 intermediate->hpll.plane = max(optimal->hpll.plane,
1332 active->hpll.plane);
1333 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1334 active->hpll.cursor);
1335 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1336 active->hpll.fbc);
1337
1338 drm_WARN_ON(display->drm,
1339 (intermediate->sr.plane >
1340 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1341 intermediate->sr.cursor >
1342 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1343 intermediate->cxsr);
1344 drm_WARN_ON(display->drm,
1345 (intermediate->sr.plane >
1346 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1347 intermediate->sr.cursor >
1348 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1349 intermediate->hpll_en);
1350
1351 drm_WARN_ON(display->drm,
1352 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1353 intermediate->fbc_en && intermediate->cxsr);
1354 drm_WARN_ON(display->drm,
1355 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1356 intermediate->fbc_en && intermediate->hpll_en);
1357
1358out:
1359 /*
1360 * If our intermediate WM are identical to the final WM, then we can
1361 * omit the post-vblank programming; only update if it's different.
1362 */
1363 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1364 new_crtc_state->wm.need_postvbl_update = true;
1365
1366 return 0;
1367}
1368
1369static int g4x_compute_watermarks(struct intel_atomic_state *state,
1370 struct intel_crtc *crtc)
1371{
1372 int ret;
1373
1374 ret = g4x_compute_pipe_wm(state, crtc);
1375 if (ret)
1376 return ret;
1377
1378 ret = g4x_compute_intermediate_wm(state, crtc);
1379 if (ret)
1380 return ret;
1381
1382 return 0;
1383}
1384
1385static void g4x_merge_wm(struct intel_display *display,
1386 struct g4x_wm_values *wm)
1387{
1388 struct intel_crtc *crtc;
1389 int num_active_pipes = 0;
1390
1391 wm->cxsr = true;
1392 wm->hpll_en = true;
1393 wm->fbc_en = true;
1394
1395 for_each_intel_crtc(display->drm, crtc) {
1396 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1397
1398 if (!crtc->active)
1399 continue;
1400
1401 if (!wm_state->cxsr)
1402 wm->cxsr = false;
1403 if (!wm_state->hpll_en)
1404 wm->hpll_en = false;
1405 if (!wm_state->fbc_en)
1406 wm->fbc_en = false;
1407
1408 num_active_pipes++;
1409 }
1410
1411 if (num_active_pipes != 1) {
1412 wm->cxsr = false;
1413 wm->hpll_en = false;
1414 wm->fbc_en = false;
1415 }
1416
1417 for_each_intel_crtc(display->drm, crtc) {
1418 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1419 enum pipe pipe = crtc->pipe;
1420
1421 wm->pipe[pipe] = wm_state->wm;
1422 if (crtc->active && wm->cxsr)
1423 wm->sr = wm_state->sr;
1424 if (crtc->active && wm->hpll_en)
1425 wm->hpll = wm_state->hpll;
1426 }
1427}
1428
1429static void g4x_program_watermarks(struct intel_display *display)
1430{
1431 struct g4x_wm_values *old_wm = &display->wm.g4x;
1432 struct g4x_wm_values new_wm = {};
1433
1434 g4x_merge_wm(display, wm: &new_wm);
1435
1436 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1437 return;
1438
1439 if (is_disabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
1440 _intel_set_memory_cxsr(display, enable: false);
1441
1442 g4x_write_wm_values(display, wm: &new_wm);
1443
1444 if (is_enabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
1445 _intel_set_memory_cxsr(display, enable: true);
1446
1447 *old_wm = new_wm;
1448}
1449
1450static void g4x_initial_watermarks(struct intel_atomic_state *state,
1451 struct intel_crtc *crtc)
1452{
1453 struct intel_display *display = to_intel_display(crtc);
1454 const struct intel_crtc_state *crtc_state =
1455 intel_atomic_get_new_crtc_state(state, crtc);
1456
1457 mutex_lock(lock: &display->wm.wm_mutex);
1458 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1459 g4x_program_watermarks(display);
1460 mutex_unlock(lock: &display->wm.wm_mutex);
1461}
1462
1463static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1464 struct intel_crtc *crtc)
1465{
1466 struct intel_display *display = to_intel_display(crtc);
1467 const struct intel_crtc_state *crtc_state =
1468 intel_atomic_get_new_crtc_state(state, crtc);
1469
1470 if (!crtc_state->wm.need_postvbl_update)
1471 return;
1472
1473 mutex_lock(lock: &display->wm.wm_mutex);
1474 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1475 g4x_program_watermarks(display);
1476 mutex_unlock(lock: &display->wm.wm_mutex);
1477}
1478
1479/* latency must be in 0.1us units. */
1480static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1481 unsigned int htotal,
1482 unsigned int width,
1483 unsigned int cpp,
1484 unsigned int latency)
1485{
1486 unsigned int ret;
1487
1488 ret = intel_wm_method2(pixel_rate, htotal,
1489 width, cpp, latency);
1490 ret = DIV_ROUND_UP(ret, 64);
1491
1492 return ret;
1493}
1494
1495static void vlv_setup_wm_latency(struct intel_display *display)
1496{
1497 /* all latencies in usec */
1498 display->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1499
1500 display->wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1501
1502 if (display->platform.cherryview) {
1503 display->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1504 display->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1505
1506 display->wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1507 }
1508}
1509
1510static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1511 const struct intel_plane_state *plane_state,
1512 int level)
1513{
1514 struct intel_display *display = to_intel_display(plane_state);
1515 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1516 const struct drm_display_mode *pipe_mode =
1517 &crtc_state->hw.pipe_mode;
1518 unsigned int pixel_rate, htotal, cpp, width, wm;
1519
1520 if (display->wm.pri_latency[level] == 0)
1521 return USHRT_MAX;
1522
1523 if (!intel_wm_plane_visible(crtc_state, plane_state))
1524 return 0;
1525
1526 cpp = plane_state->hw.fb->format->cpp[0];
1527 pixel_rate = crtc_state->pixel_rate;
1528 htotal = pipe_mode->crtc_htotal;
1529 width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1530
1531 if (plane->id == PLANE_CURSOR) {
1532 /*
1533 * FIXME the formula gives values that are
1534 * too big for the cursor FIFO, and hence we
1535 * would never be able to use cursors. For
1536 * now just hardcode the watermark.
1537 */
1538 wm = 63;
1539 } else {
1540 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1541 latency: display->wm.pri_latency[level] * 10);
1542 }
1543
1544 return min_t(unsigned int, wm, USHRT_MAX);
1545}
1546
1547static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1548{
1549 return (active_planes & (BIT(PLANE_SPRITE0) |
1550 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1551}
1552
1553static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1554{
1555 struct intel_display *display = to_intel_display(crtc_state);
1556 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1557 const struct g4x_pipe_wm *raw =
1558 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1559 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1560 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1561 int num_active_planes = hweight8(active_planes);
1562 const int fifo_size = 511;
1563 int fifo_extra, fifo_left = fifo_size;
1564 int sprite0_fifo_extra = 0;
1565 unsigned int total_rate;
1566 enum plane_id plane_id;
1567
1568 /*
1569 * When enabling sprite0 after sprite1 has already been enabled
1570 * we tend to get an underrun unless sprite0 already has some
1571 * FIFO space allocated. Hence we always allocate at least one
1572 * cacheline for sprite0 whenever sprite1 is enabled.
1573 *
1574 * All other plane enable sequences appear immune to this problem.
1575 */
1576 if (vlv_need_sprite0_fifo_workaround(active_planes))
1577 sprite0_fifo_extra = 1;
1578
1579 total_rate = raw->plane[PLANE_PRIMARY] +
1580 raw->plane[PLANE_SPRITE0] +
1581 raw->plane[PLANE_SPRITE1] +
1582 sprite0_fifo_extra;
1583
1584 if (total_rate > fifo_size)
1585 return -EINVAL;
1586
1587 if (total_rate == 0)
1588 total_rate = 1;
1589
1590 for_each_plane_id_on_crtc(crtc, plane_id) {
1591 unsigned int rate;
1592
1593 if ((active_planes & BIT(plane_id)) == 0) {
1594 fifo_state->plane[plane_id] = 0;
1595 continue;
1596 }
1597
1598 rate = raw->plane[plane_id];
1599 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1600 fifo_left -= fifo_state->plane[plane_id];
1601 }
1602
1603 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1604 fifo_left -= sprite0_fifo_extra;
1605
1606 fifo_state->plane[PLANE_CURSOR] = 63;
1607
1608 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1609
1610 /* spread the remainder evenly */
1611 for_each_plane_id_on_crtc(crtc, plane_id) {
1612 int plane_extra;
1613
1614 if (fifo_left == 0)
1615 break;
1616
1617 if ((active_planes & BIT(plane_id)) == 0)
1618 continue;
1619
1620 plane_extra = min(fifo_extra, fifo_left);
1621 fifo_state->plane[plane_id] += plane_extra;
1622 fifo_left -= plane_extra;
1623 }
1624
1625 drm_WARN_ON(display->drm, active_planes != 0 && fifo_left != 0);
1626
1627 /* give it all to the first plane if none are active */
1628 if (active_planes == 0) {
1629 drm_WARN_ON(display->drm, fifo_left != fifo_size);
1630 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1631 }
1632
1633 return 0;
1634}
1635
1636/* mark all levels starting from 'level' as invalid */
1637static void vlv_invalidate_wms(struct intel_crtc *crtc,
1638 struct vlv_wm_state *wm_state, int level)
1639{
1640 struct intel_display *display = to_intel_display(crtc);
1641
1642 for (; level < display->wm.num_levels; level++) {
1643 enum plane_id plane_id;
1644
1645 for_each_plane_id_on_crtc(crtc, plane_id)
1646 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1647
1648 wm_state->sr[level].cursor = USHRT_MAX;
1649 wm_state->sr[level].plane = USHRT_MAX;
1650 }
1651}
1652
1653static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1654{
1655 if (wm > fifo_size)
1656 return USHRT_MAX;
1657 else
1658 return fifo_size - wm;
1659}
1660
1661/*
1662 * Starting from 'level' set all higher
1663 * levels to 'value' in the "raw" watermarks.
1664 */
1665static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1666 int level, enum plane_id plane_id, u16 value)
1667{
1668 struct intel_display *display = to_intel_display(crtc_state);
1669 bool dirty = false;
1670
1671 for (; level < display->wm.num_levels; level++) {
1672 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1673
1674 dirty |= raw->plane[plane_id] != value;
1675 raw->plane[plane_id] = value;
1676 }
1677
1678 return dirty;
1679}
1680
1681static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1682 const struct intel_plane_state *plane_state)
1683{
1684 struct intel_display *display = to_intel_display(crtc_state);
1685 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1686 enum plane_id plane_id = plane->id;
1687 int level;
1688 bool dirty = false;
1689
1690 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1691 dirty |= vlv_raw_plane_wm_set(crtc_state, level: 0, plane_id, value: 0);
1692 goto out;
1693 }
1694
1695 for (level = 0; level < display->wm.num_levels; level++) {
1696 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1697 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1698 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1699
1700 if (wm > max_wm)
1701 break;
1702
1703 dirty |= raw->plane[plane_id] != wm;
1704 raw->plane[plane_id] = wm;
1705 }
1706
1707 /* mark all higher levels as invalid */
1708 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1709
1710out:
1711 if (dirty)
1712 drm_dbg_kms(display->drm,
1713 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1714 plane->base.name,
1715 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1716 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1717 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1718
1719 return dirty;
1720}
1721
1722static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1723 enum plane_id plane_id, int level)
1724{
1725 const struct g4x_pipe_wm *raw =
1726 &crtc_state->wm.vlv.raw[level];
1727 const struct vlv_fifo_state *fifo_state =
1728 &crtc_state->wm.vlv.fifo_state;
1729
1730 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1731}
1732
1733static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1734{
1735 return vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_PRIMARY, level) &&
1736 vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_SPRITE0, level) &&
1737 vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_SPRITE1, level) &&
1738 vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_CURSOR, level);
1739}
1740
1741static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1742{
1743 struct intel_display *display = to_intel_display(crtc_state);
1744 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1745 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1746 const struct vlv_fifo_state *fifo_state =
1747 &crtc_state->wm.vlv.fifo_state;
1748 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1749 int num_active_planes = hweight8(active_planes);
1750 enum plane_id plane_id;
1751 int level;
1752
1753 /* initially allow all levels */
1754 wm_state->num_levels = display->wm.num_levels;
1755 /*
1756 * Note that enabling cxsr with no primary/sprite planes
1757 * enabled can wedge the pipe. Hence we only allow cxsr
1758 * with exactly one enabled primary/sprite plane.
1759 */
1760 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1761
1762 for (level = 0; level < wm_state->num_levels; level++) {
1763 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1764 const int sr_fifo_size = INTEL_NUM_PIPES(display) * 512 - 1;
1765
1766 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1767 break;
1768
1769 for_each_plane_id_on_crtc(crtc, plane_id) {
1770 wm_state->wm[level].plane[plane_id] =
1771 vlv_invert_wm_value(wm: raw->plane[plane_id],
1772 fifo_size: fifo_state->plane[plane_id]);
1773 }
1774
1775 wm_state->sr[level].plane =
1776 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1777 raw->plane[PLANE_SPRITE0],
1778 raw->plane[PLANE_SPRITE1]),
1779 fifo_size: sr_fifo_size);
1780
1781 wm_state->sr[level].cursor =
1782 vlv_invert_wm_value(wm: raw->plane[PLANE_CURSOR],
1783 fifo_size: 63);
1784 }
1785
1786 if (level == 0)
1787 return -EINVAL;
1788
1789 /* limit to only levels we can actually handle */
1790 wm_state->num_levels = level;
1791
1792 /* invalidate the higher levels */
1793 vlv_invalidate_wms(crtc, wm_state, level);
1794
1795 return 0;
1796}
1797
1798static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1799 struct intel_crtc *crtc)
1800{
1801 struct intel_crtc_state *crtc_state =
1802 intel_atomic_get_new_crtc_state(state, crtc);
1803 const struct intel_plane_state *old_plane_state;
1804 const struct intel_plane_state *new_plane_state;
1805 struct intel_plane *plane;
1806 unsigned int dirty = 0;
1807 int i;
1808
1809 for_each_oldnew_intel_plane_in_state(state, plane,
1810 old_plane_state,
1811 new_plane_state, i) {
1812 if (new_plane_state->hw.crtc != &crtc->base &&
1813 old_plane_state->hw.crtc != &crtc->base)
1814 continue;
1815
1816 if (vlv_raw_plane_wm_compute(crtc_state, plane_state: new_plane_state))
1817 dirty |= BIT(plane->id);
1818 }
1819
1820 /*
1821 * DSPARB registers may have been reset due to the
1822 * power well being turned off. Make sure we restore
1823 * them to a consistent state even if no primary/sprite
1824 * planes are initially active. We also force a FIFO
1825 * recomputation so that we are sure to sanitize the
1826 * FIFO setting we took over from the BIOS even if there
1827 * are no active planes on the crtc.
1828 */
1829 if (intel_crtc_needs_modeset(crtc_state))
1830 dirty = ~0;
1831
1832 if (!dirty)
1833 return 0;
1834
1835 /* cursor changes don't warrant a FIFO recompute */
1836 if (dirty & ~BIT(PLANE_CURSOR)) {
1837 const struct intel_crtc_state *old_crtc_state =
1838 intel_atomic_get_old_crtc_state(state, crtc);
1839 const struct vlv_fifo_state *old_fifo_state =
1840 &old_crtc_state->wm.vlv.fifo_state;
1841 const struct vlv_fifo_state *new_fifo_state =
1842 &crtc_state->wm.vlv.fifo_state;
1843 int ret;
1844
1845 ret = vlv_compute_fifo(crtc_state);
1846 if (ret)
1847 return ret;
1848
1849 if (intel_crtc_needs_modeset(crtc_state) ||
1850 memcmp(old_fifo_state, new_fifo_state,
1851 sizeof(*new_fifo_state)) != 0)
1852 crtc_state->fifo_changed = true;
1853 }
1854
1855 return _vlv_compute_pipe_wm(crtc_state);
1856}
1857
1858#define VLV_FIFO(plane, value) \
1859 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1860
1861static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1862 struct intel_crtc *crtc)
1863{
1864 struct intel_display *display = to_intel_display(crtc);
1865 struct drm_i915_private *dev_priv = to_i915(dev: crtc->base.dev);
1866 struct intel_uncore *uncore = &dev_priv->uncore;
1867 const struct intel_crtc_state *crtc_state =
1868 intel_atomic_get_new_crtc_state(state, crtc);
1869 const struct vlv_fifo_state *fifo_state =
1870 &crtc_state->wm.vlv.fifo_state;
1871 int sprite0_start, sprite1_start, fifo_size;
1872 u32 dsparb, dsparb2, dsparb3;
1873
1874 if (!crtc_state->fifo_changed)
1875 return;
1876
1877 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1878 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1879 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1880
1881 drm_WARN_ON(display->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1882 drm_WARN_ON(display->drm, fifo_size != 511);
1883
1884 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1885
1886 /*
1887 * uncore.lock serves a double purpose here. It allows us to
1888 * use the less expensive I915_{READ,WRITE}_FW() functions, and
1889 * it protects the DSPARB registers from getting clobbered by
1890 * parallel updates from multiple pipes.
1891 *
1892 * intel_pipe_update_start() has already disabled interrupts
1893 * for us, so a plain spin_lock() is sufficient here.
1894 */
1895 spin_lock(lock: &uncore->lock);
1896
1897 switch (crtc->pipe) {
1898 case PIPE_A:
1899 dsparb = intel_de_read_fw(display, DSPARB(display));
1900 dsparb2 = intel_de_read_fw(display, DSPARB2);
1901
1902 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1903 VLV_FIFO(SPRITEB, 0xff));
1904 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1905 VLV_FIFO(SPRITEB, sprite1_start));
1906
1907 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1908 VLV_FIFO(SPRITEB_HI, 0x1));
1909 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1910 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1911
1912 intel_de_write_fw(display, DSPARB(display), val: dsparb);
1913 intel_de_write_fw(display, DSPARB2, val: dsparb2);
1914 break;
1915 case PIPE_B:
1916 dsparb = intel_de_read_fw(display, DSPARB(display));
1917 dsparb2 = intel_de_read_fw(display, DSPARB2);
1918
1919 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1920 VLV_FIFO(SPRITED, 0xff));
1921 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1922 VLV_FIFO(SPRITED, sprite1_start));
1923
1924 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1925 VLV_FIFO(SPRITED_HI, 0xff));
1926 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1927 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1928
1929 intel_de_write_fw(display, DSPARB(display), val: dsparb);
1930 intel_de_write_fw(display, DSPARB2, val: dsparb2);
1931 break;
1932 case PIPE_C:
1933 dsparb3 = intel_de_read_fw(display, DSPARB3);
1934 dsparb2 = intel_de_read_fw(display, DSPARB2);
1935
1936 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1937 VLV_FIFO(SPRITEF, 0xff));
1938 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1939 VLV_FIFO(SPRITEF, sprite1_start));
1940
1941 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1942 VLV_FIFO(SPRITEF_HI, 0xff));
1943 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1944 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1945
1946 intel_de_write_fw(display, DSPARB3, val: dsparb3);
1947 intel_de_write_fw(display, DSPARB2, val: dsparb2);
1948 break;
1949 default:
1950 break;
1951 }
1952
1953 intel_de_read_fw(display, DSPARB(display));
1954
1955 spin_unlock(lock: &uncore->lock);
1956}
1957
1958#undef VLV_FIFO
1959
1960static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1961 struct intel_crtc *crtc)
1962{
1963 struct intel_crtc_state *new_crtc_state =
1964 intel_atomic_get_new_crtc_state(state, crtc);
1965 const struct intel_crtc_state *old_crtc_state =
1966 intel_atomic_get_old_crtc_state(state, crtc);
1967 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1968 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1969 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1970 int level;
1971
1972 if (!new_crtc_state->hw.active ||
1973 intel_crtc_needs_modeset(crtc_state: new_crtc_state)) {
1974 *intermediate = *optimal;
1975
1976 intermediate->cxsr = false;
1977 goto out;
1978 }
1979
1980 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1981 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1982 !new_crtc_state->disable_cxsr;
1983
1984 for (level = 0; level < intermediate->num_levels; level++) {
1985 enum plane_id plane_id;
1986
1987 for_each_plane_id_on_crtc(crtc, plane_id) {
1988 intermediate->wm[level].plane[plane_id] =
1989 min(optimal->wm[level].plane[plane_id],
1990 active->wm[level].plane[plane_id]);
1991 }
1992
1993 intermediate->sr[level].plane = min(optimal->sr[level].plane,
1994 active->sr[level].plane);
1995 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1996 active->sr[level].cursor);
1997 }
1998
1999 vlv_invalidate_wms(crtc, wm_state: intermediate, level);
2000
2001out:
2002 /*
2003 * If our intermediate WM are identical to the final WM, then we can
2004 * omit the post-vblank programming; only update if it's different.
2005 */
2006 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2007 new_crtc_state->wm.need_postvbl_update = true;
2008
2009 return 0;
2010}
2011
2012static int vlv_compute_watermarks(struct intel_atomic_state *state,
2013 struct intel_crtc *crtc)
2014{
2015 int ret;
2016
2017 ret = vlv_compute_pipe_wm(state, crtc);
2018 if (ret)
2019 return ret;
2020
2021 ret = vlv_compute_intermediate_wm(state, crtc);
2022 if (ret)
2023 return ret;
2024
2025 return 0;
2026}
2027
2028static void vlv_merge_wm(struct intel_display *display,
2029 struct vlv_wm_values *wm)
2030{
2031 struct intel_crtc *crtc;
2032 int num_active_pipes = 0;
2033
2034 wm->level = display->wm.num_levels - 1;
2035 wm->cxsr = true;
2036
2037 for_each_intel_crtc(display->drm, crtc) {
2038 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2039
2040 if (!crtc->active)
2041 continue;
2042
2043 if (!wm_state->cxsr)
2044 wm->cxsr = false;
2045
2046 num_active_pipes++;
2047 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2048 }
2049
2050 if (num_active_pipes != 1)
2051 wm->cxsr = false;
2052
2053 if (num_active_pipes > 1)
2054 wm->level = VLV_WM_LEVEL_PM2;
2055
2056 for_each_intel_crtc(display->drm, crtc) {
2057 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2058 enum pipe pipe = crtc->pipe;
2059
2060 wm->pipe[pipe] = wm_state->wm[wm->level];
2061 if (crtc->active && wm->cxsr)
2062 wm->sr = wm_state->sr[wm->level];
2063
2064 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2065 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2066 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2067 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2068 }
2069}
2070
2071static void vlv_program_watermarks(struct intel_display *display)
2072{
2073 struct vlv_wm_values *old_wm = &display->wm.vlv;
2074 struct vlv_wm_values new_wm = {};
2075
2076 vlv_merge_wm(display, wm: &new_wm);
2077
2078 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2079 return;
2080
2081 if (is_disabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_DDR_DVFS))
2082 chv_set_memory_dvfs(display, enable: false);
2083
2084 if (is_disabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_PM5))
2085 chv_set_memory_pm5(display, enable: false);
2086
2087 if (is_disabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
2088 _intel_set_memory_cxsr(display, enable: false);
2089
2090 vlv_write_wm_values(display, wm: &new_wm);
2091
2092 if (is_enabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
2093 _intel_set_memory_cxsr(display, enable: true);
2094
2095 if (is_enabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_PM5))
2096 chv_set_memory_pm5(display, enable: true);
2097
2098 if (is_enabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_DDR_DVFS))
2099 chv_set_memory_dvfs(display, enable: true);
2100
2101 *old_wm = new_wm;
2102}
2103
2104static void vlv_initial_watermarks(struct intel_atomic_state *state,
2105 struct intel_crtc *crtc)
2106{
2107 struct intel_display *display = to_intel_display(crtc);
2108 const struct intel_crtc_state *crtc_state =
2109 intel_atomic_get_new_crtc_state(state, crtc);
2110
2111 mutex_lock(lock: &display->wm.wm_mutex);
2112 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2113 vlv_program_watermarks(display);
2114 mutex_unlock(lock: &display->wm.wm_mutex);
2115}
2116
2117static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2118 struct intel_crtc *crtc)
2119{
2120 struct intel_display *display = to_intel_display(crtc);
2121 const struct intel_crtc_state *crtc_state =
2122 intel_atomic_get_new_crtc_state(state, crtc);
2123
2124 if (!crtc_state->wm.need_postvbl_update)
2125 return;
2126
2127 mutex_lock(lock: &display->wm.wm_mutex);
2128 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2129 vlv_program_watermarks(display);
2130 mutex_unlock(lock: &display->wm.wm_mutex);
2131}
2132
2133static void i965_update_wm(struct intel_display *display)
2134{
2135 struct intel_crtc *crtc;
2136 int srwm = 1;
2137 int cursor_sr = 16;
2138 bool cxsr_enabled;
2139
2140 /* Calc sr entries for one plane configs */
2141 crtc = single_enabled_crtc(display);
2142 if (crtc) {
2143 /* self-refresh has much higher latency */
2144 static const int sr_latency_ns = 12000;
2145 const struct drm_display_mode *pipe_mode =
2146 &crtc->config->hw.pipe_mode;
2147 const struct drm_framebuffer *fb =
2148 crtc->base.primary->state->fb;
2149 int pixel_rate = crtc->config->pixel_rate;
2150 int htotal = pipe_mode->crtc_htotal;
2151 int width = drm_rect_width(r: &crtc->base.primary->state->src) >> 16;
2152 int cpp = fb->format->cpp[0];
2153 int entries;
2154
2155 entries = intel_wm_method2(pixel_rate, htotal,
2156 width, cpp, latency: sr_latency_ns / 100);
2157 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2158 srwm = I965_FIFO_SIZE - entries;
2159 if (srwm < 0)
2160 srwm = 1;
2161 srwm &= 0x1ff;
2162 drm_dbg_kms(display->drm,
2163 "self-refresh entries: %d, wm: %d\n",
2164 entries, srwm);
2165
2166 entries = intel_wm_method2(pixel_rate, htotal,
2167 width: crtc->base.cursor->state->crtc_w, cpp: 4,
2168 latency: sr_latency_ns / 100);
2169 entries = DIV_ROUND_UP(entries,
2170 i965_cursor_wm_info.cacheline_size) +
2171 i965_cursor_wm_info.guard_size;
2172
2173 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2174 if (cursor_sr > i965_cursor_wm_info.max_wm)
2175 cursor_sr = i965_cursor_wm_info.max_wm;
2176
2177 drm_dbg_kms(display->drm,
2178 "self-refresh watermark: display plane %d "
2179 "cursor %d\n", srwm, cursor_sr);
2180
2181 cxsr_enabled = true;
2182 } else {
2183 cxsr_enabled = false;
2184 /* Turn off self refresh if both pipes are enabled */
2185 intel_set_memory_cxsr(display, enable: false);
2186 }
2187
2188 drm_dbg_kms(display->drm,
2189 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2190 srwm);
2191
2192 /* 965 has limitations... */
2193 intel_de_write(display, DSPFW1(display),
2194 FW_WM(srwm, SR) |
2195 FW_WM(8, CURSORB) |
2196 FW_WM(8, PLANEB) |
2197 FW_WM(8, PLANEA));
2198 intel_de_write(display, DSPFW2(display),
2199 FW_WM(8, CURSORA) |
2200 FW_WM(8, PLANEC_OLD));
2201 /* update cursor SR watermark */
2202 intel_de_write(display, DSPFW3(display),
2203 FW_WM(cursor_sr, CURSOR_SR));
2204
2205 if (cxsr_enabled)
2206 intel_set_memory_cxsr(display, enable: true);
2207}
2208
2209#undef FW_WM
2210
2211static struct intel_crtc *intel_crtc_for_plane(struct intel_display *display,
2212 enum i9xx_plane_id i9xx_plane)
2213{
2214 struct intel_plane *plane;
2215
2216 for_each_intel_plane(display->drm, plane) {
2217 if (plane->id == PLANE_PRIMARY &&
2218 plane->i9xx_plane == i9xx_plane)
2219 return intel_crtc_for_pipe(display, pipe: plane->pipe);
2220 }
2221
2222 return NULL;
2223}
2224
2225static void i9xx_update_wm(struct intel_display *display)
2226{
2227 const struct intel_watermark_params *wm_info;
2228 u32 fwater_lo;
2229 u32 fwater_hi;
2230 int cwm, srwm = 1;
2231 int fifo_size;
2232 int planea_wm, planeb_wm;
2233 struct intel_crtc *crtc;
2234
2235 if (display->platform.i945gm)
2236 wm_info = &i945_wm_info;
2237 else if (DISPLAY_VER(display) != 2)
2238 wm_info = &i915_wm_info;
2239 else
2240 wm_info = &i830_a_wm_info;
2241
2242 if (DISPLAY_VER(display) == 2)
2243 fifo_size = i830_get_fifo_size(display, i9xx_plane: PLANE_A);
2244 else
2245 fifo_size = i9xx_get_fifo_size(display, i9xx_plane: PLANE_A);
2246 crtc = intel_crtc_for_plane(display, i9xx_plane: PLANE_A);
2247 if (intel_crtc_active(crtc)) {
2248 const struct drm_framebuffer *fb =
2249 crtc->base.primary->state->fb;
2250 int cpp;
2251
2252 if (DISPLAY_VER(display) == 2)
2253 cpp = 4;
2254 else
2255 cpp = fb->format->cpp[0];
2256
2257 planea_wm = intel_calculate_wm(display, pixel_rate: crtc->config->pixel_rate,
2258 wm: wm_info, fifo_size, cpp,
2259 latency_ns: pessimal_latency_ns);
2260 } else {
2261 planea_wm = fifo_size - wm_info->guard_size;
2262 if (planea_wm > (long)wm_info->max_wm)
2263 planea_wm = wm_info->max_wm;
2264 }
2265
2266 if (DISPLAY_VER(display) == 2)
2267 wm_info = &i830_bc_wm_info;
2268
2269 if (DISPLAY_VER(display) == 2)
2270 fifo_size = i830_get_fifo_size(display, i9xx_plane: PLANE_B);
2271 else
2272 fifo_size = i9xx_get_fifo_size(display, i9xx_plane: PLANE_B);
2273 crtc = intel_crtc_for_plane(display, i9xx_plane: PLANE_B);
2274 if (intel_crtc_active(crtc)) {
2275 const struct drm_framebuffer *fb =
2276 crtc->base.primary->state->fb;
2277 int cpp;
2278
2279 if (DISPLAY_VER(display) == 2)
2280 cpp = 4;
2281 else
2282 cpp = fb->format->cpp[0];
2283
2284 planeb_wm = intel_calculate_wm(display, pixel_rate: crtc->config->pixel_rate,
2285 wm: wm_info, fifo_size, cpp,
2286 latency_ns: pessimal_latency_ns);
2287 } else {
2288 planeb_wm = fifo_size - wm_info->guard_size;
2289 if (planeb_wm > (long)wm_info->max_wm)
2290 planeb_wm = wm_info->max_wm;
2291 }
2292
2293 drm_dbg_kms(display->drm,
2294 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2295
2296 crtc = single_enabled_crtc(display);
2297 if (display->platform.i915gm && crtc) {
2298 struct drm_gem_object *obj;
2299
2300 obj = intel_fb_bo(fb: crtc->base.primary->state->fb);
2301
2302 /* self-refresh seems busted with untiled */
2303 if (!intel_bo_is_tiled(obj))
2304 crtc = NULL;
2305 }
2306
2307 /*
2308 * Overlay gets an aggressive default since video jitter is bad.
2309 */
2310 cwm = 2;
2311
2312 /* Play safe and disable self-refresh before adjusting watermarks. */
2313 intel_set_memory_cxsr(display, enable: false);
2314
2315 /* Calc sr entries for one plane configs */
2316 if (HAS_FW_BLC(display) && crtc) {
2317 /* self-refresh has much higher latency */
2318 static const int sr_latency_ns = 6000;
2319 const struct drm_display_mode *pipe_mode =
2320 &crtc->config->hw.pipe_mode;
2321 const struct drm_framebuffer *fb =
2322 crtc->base.primary->state->fb;
2323 int pixel_rate = crtc->config->pixel_rate;
2324 int htotal = pipe_mode->crtc_htotal;
2325 int width = drm_rect_width(r: &crtc->base.primary->state->src) >> 16;
2326 int cpp;
2327 int entries;
2328
2329 if (display->platform.i915gm || display->platform.i945gm)
2330 cpp = 4;
2331 else
2332 cpp = fb->format->cpp[0];
2333
2334 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2335 latency: sr_latency_ns / 100);
2336 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2337 drm_dbg_kms(display->drm,
2338 "self-refresh entries: %d\n", entries);
2339 srwm = wm_info->fifo_size - entries;
2340 if (srwm < 0)
2341 srwm = 1;
2342
2343 if (display->platform.i945g || display->platform.i945gm)
2344 intel_de_write(display, FW_BLC_SELF,
2345 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2346 else
2347 intel_de_write(display, FW_BLC_SELF, val: srwm & 0x3f);
2348 }
2349
2350 drm_dbg_kms(display->drm,
2351 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2352 planea_wm, planeb_wm, cwm, srwm);
2353
2354 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2355 fwater_hi = (cwm & 0x1f);
2356
2357 /* Set request length to 8 cachelines per fetch */
2358 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2359 fwater_hi = fwater_hi | (1 << 8);
2360
2361 intel_de_write(display, FW_BLC, val: fwater_lo);
2362 intel_de_write(display, FW_BLC2, val: fwater_hi);
2363
2364 if (crtc)
2365 intel_set_memory_cxsr(display, enable: true);
2366}
2367
2368static void i845_update_wm(struct intel_display *display)
2369{
2370 struct intel_crtc *crtc;
2371 u32 fwater_lo;
2372 int planea_wm;
2373
2374 crtc = single_enabled_crtc(display);
2375 if (crtc == NULL)
2376 return;
2377
2378 planea_wm = intel_calculate_wm(display, pixel_rate: crtc->config->pixel_rate,
2379 wm: &i845_wm_info,
2380 fifo_size: i845_get_fifo_size(display, i9xx_plane: PLANE_A),
2381 cpp: 4, latency_ns: pessimal_latency_ns);
2382 fwater_lo = intel_de_read(display, FW_BLC) & ~0xfff;
2383 fwater_lo |= (3<<8) | planea_wm;
2384
2385 drm_dbg_kms(display->drm,
2386 "Setting FIFO watermarks - A: %d\n", planea_wm);
2387
2388 intel_de_write(display, FW_BLC, val: fwater_lo);
2389}
2390
2391/* latency must be in 0.1us units. */
2392static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2393 unsigned int cpp,
2394 unsigned int latency)
2395{
2396 unsigned int ret;
2397
2398 ret = intel_wm_method1(pixel_rate, cpp, latency);
2399 ret = DIV_ROUND_UP(ret, 64) + 2;
2400
2401 return ret;
2402}
2403
2404/* latency must be in 0.1us units. */
2405static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2406 unsigned int htotal,
2407 unsigned int width,
2408 unsigned int cpp,
2409 unsigned int latency)
2410{
2411 unsigned int ret;
2412
2413 ret = intel_wm_method2(pixel_rate, htotal,
2414 width, cpp, latency);
2415 ret = DIV_ROUND_UP(ret, 64) + 2;
2416
2417 return ret;
2418}
2419
2420static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2421{
2422 /*
2423 * Neither of these should be possible since this function shouldn't be
2424 * called if the CRTC is off or the plane is invisible. But let's be
2425 * extra paranoid to avoid a potential divide-by-zero if we screw up
2426 * elsewhere in the driver.
2427 */
2428 if (WARN_ON(!cpp))
2429 return 0;
2430 if (WARN_ON(!horiz_pixels))
2431 return 0;
2432
2433 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2434}
2435
2436struct ilk_wm_maximums {
2437 u16 pri;
2438 u16 spr;
2439 u16 cur;
2440 u16 fbc;
2441};
2442
2443/*
2444 * For both WM_PIPE and WM_LP.
2445 * mem_value must be in 0.1us units.
2446 */
2447static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2448 const struct intel_plane_state *plane_state,
2449 u32 mem_value, bool is_lp)
2450{
2451 u32 method1, method2;
2452 int cpp;
2453
2454 if (mem_value == 0)
2455 return U32_MAX;
2456
2457 if (!intel_wm_plane_visible(crtc_state, plane_state))
2458 return 0;
2459
2460 cpp = plane_state->hw.fb->format->cpp[0];
2461
2462 method1 = ilk_wm_method1(pixel_rate: crtc_state->pixel_rate, cpp, latency: mem_value);
2463
2464 if (!is_lp)
2465 return method1;
2466
2467 method2 = ilk_wm_method2(pixel_rate: crtc_state->pixel_rate,
2468 htotal: crtc_state->hw.pipe_mode.crtc_htotal,
2469 width: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2470 cpp, latency: mem_value);
2471
2472 return min(method1, method2);
2473}
2474
2475/*
2476 * For both WM_PIPE and WM_LP.
2477 * mem_value must be in 0.1us units.
2478 */
2479static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2480 const struct intel_plane_state *plane_state,
2481 u32 mem_value)
2482{
2483 u32 method1, method2;
2484 int cpp;
2485
2486 if (mem_value == 0)
2487 return U32_MAX;
2488
2489 if (!intel_wm_plane_visible(crtc_state, plane_state))
2490 return 0;
2491
2492 cpp = plane_state->hw.fb->format->cpp[0];
2493
2494 method1 = ilk_wm_method1(pixel_rate: crtc_state->pixel_rate, cpp, latency: mem_value);
2495 method2 = ilk_wm_method2(pixel_rate: crtc_state->pixel_rate,
2496 htotal: crtc_state->hw.pipe_mode.crtc_htotal,
2497 width: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2498 cpp, latency: mem_value);
2499 return min(method1, method2);
2500}
2501
2502/*
2503 * For both WM_PIPE and WM_LP.
2504 * mem_value must be in 0.1us units.
2505 */
2506static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2507 const struct intel_plane_state *plane_state,
2508 u32 mem_value)
2509{
2510 int cpp;
2511
2512 if (mem_value == 0)
2513 return U32_MAX;
2514
2515 if (!intel_wm_plane_visible(crtc_state, plane_state))
2516 return 0;
2517
2518 cpp = plane_state->hw.fb->format->cpp[0];
2519
2520 return ilk_wm_method2(pixel_rate: crtc_state->pixel_rate,
2521 htotal: crtc_state->hw.pipe_mode.crtc_htotal,
2522 width: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2523 cpp, latency: mem_value);
2524}
2525
2526/* Only for WM_LP. */
2527static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2528 const struct intel_plane_state *plane_state,
2529 u32 pri_val)
2530{
2531 int cpp;
2532
2533 if (!intel_wm_plane_visible(crtc_state, plane_state))
2534 return 0;
2535
2536 cpp = plane_state->hw.fb->format->cpp[0];
2537
2538 return ilk_wm_fbc(pri_val, horiz_pixels: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2539 cpp);
2540}
2541
2542static unsigned int
2543ilk_display_fifo_size(struct intel_display *display)
2544{
2545 if (DISPLAY_VER(display) >= 8)
2546 return 3072;
2547 else if (DISPLAY_VER(display) >= 7)
2548 return 768;
2549 else
2550 return 512;
2551}
2552
2553static unsigned int
2554ilk_plane_wm_reg_max(struct intel_display *display,
2555 int level, bool is_sprite)
2556{
2557 if (DISPLAY_VER(display) >= 8)
2558 /* BDW primary/sprite plane watermarks */
2559 return level == 0 ? 255 : 2047;
2560 else if (DISPLAY_VER(display) >= 7)
2561 /* IVB/HSW primary/sprite plane watermarks */
2562 return level == 0 ? 127 : 1023;
2563 else if (!is_sprite)
2564 /* ILK/SNB primary plane watermarks */
2565 return level == 0 ? 127 : 511;
2566 else
2567 /* ILK/SNB sprite plane watermarks */
2568 return level == 0 ? 63 : 255;
2569}
2570
2571static unsigned int
2572ilk_cursor_wm_reg_max(struct intel_display *display, int level)
2573{
2574 if (DISPLAY_VER(display) >= 7)
2575 return level == 0 ? 63 : 255;
2576 else
2577 return level == 0 ? 31 : 63;
2578}
2579
2580static unsigned int ilk_fbc_wm_reg_max(struct intel_display *display)
2581{
2582 if (DISPLAY_VER(display) >= 8)
2583 return 31;
2584 else
2585 return 15;
2586}
2587
2588/* Calculate the maximum primary/sprite plane watermark */
2589static unsigned int ilk_plane_wm_max(struct intel_display *display,
2590 int level,
2591 const struct intel_wm_config *config,
2592 enum intel_ddb_partitioning ddb_partitioning,
2593 bool is_sprite)
2594{
2595 unsigned int fifo_size = ilk_display_fifo_size(display);
2596
2597 /* if sprites aren't enabled, sprites get nothing */
2598 if (is_sprite && !config->sprites_enabled)
2599 return 0;
2600
2601 /* HSW allows LP1+ watermarks even with multiple pipes */
2602 if (level == 0 || config->num_pipes_active > 1) {
2603 fifo_size /= INTEL_NUM_PIPES(display);
2604
2605 /*
2606 * For some reason the non self refresh
2607 * FIFO size is only half of the self
2608 * refresh FIFO size on ILK/SNB.
2609 */
2610 if (DISPLAY_VER(display) < 7)
2611 fifo_size /= 2;
2612 }
2613
2614 if (config->sprites_enabled) {
2615 /* level 0 is always calculated with 1:1 split */
2616 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2617 if (is_sprite)
2618 fifo_size *= 5;
2619 fifo_size /= 6;
2620 } else {
2621 fifo_size /= 2;
2622 }
2623 }
2624
2625 /* clamp to max that the registers can hold */
2626 return min(fifo_size, ilk_plane_wm_reg_max(display, level, is_sprite));
2627}
2628
2629/* Calculate the maximum cursor plane watermark */
2630static unsigned int ilk_cursor_wm_max(struct intel_display *display,
2631 int level,
2632 const struct intel_wm_config *config)
2633{
2634 /* HSW LP1+ watermarks w/ multiple pipes */
2635 if (level > 0 && config->num_pipes_active > 1)
2636 return 64;
2637
2638 /* otherwise just report max that registers can hold */
2639 return ilk_cursor_wm_reg_max(display, level);
2640}
2641
2642static void ilk_compute_wm_maximums(struct intel_display *display,
2643 int level,
2644 const struct intel_wm_config *config,
2645 enum intel_ddb_partitioning ddb_partitioning,
2646 struct ilk_wm_maximums *max)
2647{
2648 max->pri = ilk_plane_wm_max(display, level, config, ddb_partitioning, is_sprite: false);
2649 max->spr = ilk_plane_wm_max(display, level, config, ddb_partitioning, is_sprite: true);
2650 max->cur = ilk_cursor_wm_max(display, level, config);
2651 max->fbc = ilk_fbc_wm_reg_max(display);
2652}
2653
2654static void ilk_compute_wm_reg_maximums(struct intel_display *display,
2655 int level,
2656 struct ilk_wm_maximums *max)
2657{
2658 max->pri = ilk_plane_wm_reg_max(display, level, is_sprite: false);
2659 max->spr = ilk_plane_wm_reg_max(display, level, is_sprite: true);
2660 max->cur = ilk_cursor_wm_reg_max(display, level);
2661 max->fbc = ilk_fbc_wm_reg_max(display);
2662}
2663
2664static bool ilk_validate_wm_level(struct intel_display *display,
2665 int level,
2666 const struct ilk_wm_maximums *max,
2667 struct intel_wm_level *result)
2668{
2669 bool ret;
2670
2671 /* already determined to be invalid? */
2672 if (!result->enable)
2673 return false;
2674
2675 result->enable = result->pri_val <= max->pri &&
2676 result->spr_val <= max->spr &&
2677 result->cur_val <= max->cur;
2678
2679 ret = result->enable;
2680
2681 /*
2682 * HACK until we can pre-compute everything,
2683 * and thus fail gracefully if LP0 watermarks
2684 * are exceeded...
2685 */
2686 if (level == 0 && !result->enable) {
2687 if (result->pri_val > max->pri)
2688 drm_dbg_kms(display->drm,
2689 "Primary WM%d too large %u (max %u)\n",
2690 level, result->pri_val, max->pri);
2691 if (result->spr_val > max->spr)
2692 drm_dbg_kms(display->drm,
2693 "Sprite WM%d too large %u (max %u)\n",
2694 level, result->spr_val, max->spr);
2695 if (result->cur_val > max->cur)
2696 drm_dbg_kms(display->drm,
2697 "Cursor WM%d too large %u (max %u)\n",
2698 level, result->cur_val, max->cur);
2699
2700 result->pri_val = min_t(u32, result->pri_val, max->pri);
2701 result->spr_val = min_t(u32, result->spr_val, max->spr);
2702 result->cur_val = min_t(u32, result->cur_val, max->cur);
2703 result->enable = true;
2704 }
2705
2706 return ret;
2707}
2708
2709static void ilk_compute_wm_level(struct intel_display *display,
2710 const struct intel_crtc *crtc,
2711 int level,
2712 struct intel_crtc_state *crtc_state,
2713 const struct intel_plane_state *pristate,
2714 const struct intel_plane_state *sprstate,
2715 const struct intel_plane_state *curstate,
2716 struct intel_wm_level *result)
2717{
2718 u16 pri_latency = display->wm.pri_latency[level];
2719 u16 spr_latency = display->wm.spr_latency[level];
2720 u16 cur_latency = display->wm.cur_latency[level];
2721
2722 /* WM1+ latency values stored in 0.5us units */
2723 if (level > 0) {
2724 pri_latency *= 5;
2725 spr_latency *= 5;
2726 cur_latency *= 5;
2727 }
2728
2729 if (pristate) {
2730 result->pri_val = ilk_compute_pri_wm(crtc_state, plane_state: pristate,
2731 mem_value: pri_latency, is_lp: level);
2732 result->fbc_val = ilk_compute_fbc_wm(crtc_state, plane_state: pristate, pri_val: result->pri_val);
2733 }
2734
2735 if (sprstate)
2736 result->spr_val = ilk_compute_spr_wm(crtc_state, plane_state: sprstate, mem_value: spr_latency);
2737
2738 if (curstate)
2739 result->cur_val = ilk_compute_cur_wm(crtc_state, plane_state: curstate, mem_value: cur_latency);
2740
2741 result->enable = true;
2742}
2743
2744static void hsw_read_wm_latency(struct intel_display *display, u16 wm[])
2745{
2746 struct drm_i915_private *i915 = to_i915(dev: display->drm);
2747 u64 sskpd;
2748
2749 display->wm.num_levels = 5;
2750
2751 sskpd = intel_uncore_read64(uncore: &i915->uncore, MCH_SSKPD);
2752
2753 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2754 if (wm[0] == 0)
2755 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2756 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2757 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2758 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2759 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2760}
2761
2762static void snb_read_wm_latency(struct intel_display *display, u16 wm[])
2763{
2764 struct drm_i915_private *i915 = to_i915(dev: display->drm);
2765 u32 sskpd;
2766
2767 display->wm.num_levels = 4;
2768
2769 sskpd = intel_uncore_read(uncore: &i915->uncore, MCH_SSKPD);
2770
2771 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2772 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2773 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2774 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2775}
2776
2777static void ilk_read_wm_latency(struct intel_display *display, u16 wm[])
2778{
2779 struct drm_i915_private *i915 = to_i915(dev: display->drm);
2780 u32 mltr;
2781
2782 display->wm.num_levels = 3;
2783
2784 mltr = intel_uncore_read(uncore: &i915->uncore, MLTR_ILK);
2785
2786 /* ILK primary LP0 latency is 700 ns */
2787 wm[0] = 7;
2788 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2789 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2790}
2791
2792static void intel_fixup_spr_wm_latency(struct intel_display *display, u16 wm[5])
2793{
2794 /* ILK sprite LP0 latency is 1300 ns */
2795 if (DISPLAY_VER(display) == 5)
2796 wm[0] = 13;
2797}
2798
2799static void intel_fixup_cur_wm_latency(struct intel_display *display, u16 wm[5])
2800{
2801 /* ILK cursor LP0 latency is 1300 ns */
2802 if (DISPLAY_VER(display) == 5)
2803 wm[0] = 13;
2804}
2805
2806static bool ilk_increase_wm_latency(struct intel_display *display, u16 wm[5], u16 min)
2807{
2808 int level;
2809
2810 if (wm[0] >= min)
2811 return false;
2812
2813 wm[0] = max(wm[0], min);
2814 for (level = 1; level < display->wm.num_levels; level++)
2815 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2816
2817 return true;
2818}
2819
2820static void snb_wm_latency_quirk(struct intel_display *display)
2821{
2822 bool changed;
2823
2824 /*
2825 * The BIOS provided WM memory latency values are often
2826 * inadequate for high resolution displays. Adjust them.
2827 */
2828 changed = ilk_increase_wm_latency(display, wm: display->wm.pri_latency, min: 12);
2829 changed |= ilk_increase_wm_latency(display, wm: display->wm.spr_latency, min: 12);
2830 changed |= ilk_increase_wm_latency(display, wm: display->wm.cur_latency, min: 12);
2831
2832 if (!changed)
2833 return;
2834
2835 drm_dbg_kms(display->drm,
2836 "WM latency values increased to avoid potential underruns\n");
2837 intel_print_wm_latency(display, name: "Primary", wm: display->wm.pri_latency);
2838 intel_print_wm_latency(display, name: "Sprite", wm: display->wm.spr_latency);
2839 intel_print_wm_latency(display, name: "Cursor", wm: display->wm.cur_latency);
2840}
2841
2842static void snb_wm_lp3_irq_quirk(struct intel_display *display)
2843{
2844 /*
2845 * On some SNB machines (Thinkpad X220 Tablet at least)
2846 * LP3 usage can cause vblank interrupts to be lost.
2847 * The DEIIR bit will go high but it looks like the CPU
2848 * never gets interrupted.
2849 *
2850 * It's not clear whether other interrupt source could
2851 * be affected or if this is somehow limited to vblank
2852 * interrupts only. To play it safe we disable LP3
2853 * watermarks entirely.
2854 */
2855 if (display->wm.pri_latency[3] == 0 &&
2856 display->wm.spr_latency[3] == 0 &&
2857 display->wm.cur_latency[3] == 0)
2858 return;
2859
2860 display->wm.pri_latency[3] = 0;
2861 display->wm.spr_latency[3] = 0;
2862 display->wm.cur_latency[3] = 0;
2863
2864 drm_dbg_kms(display->drm,
2865 "LP3 watermarks disabled due to potential for lost interrupts\n");
2866 intel_print_wm_latency(display, name: "Primary", wm: display->wm.pri_latency);
2867 intel_print_wm_latency(display, name: "Sprite", wm: display->wm.spr_latency);
2868 intel_print_wm_latency(display, name: "Cursor", wm: display->wm.cur_latency);
2869}
2870
2871static void ilk_setup_wm_latency(struct intel_display *display)
2872{
2873 if (display->platform.broadwell || display->platform.haswell)
2874 hsw_read_wm_latency(display, wm: display->wm.pri_latency);
2875 else if (DISPLAY_VER(display) >= 6)
2876 snb_read_wm_latency(display, wm: display->wm.pri_latency);
2877 else
2878 ilk_read_wm_latency(display, wm: display->wm.pri_latency);
2879
2880 memcpy(to: display->wm.spr_latency, from: display->wm.pri_latency,
2881 len: sizeof(display->wm.pri_latency));
2882 memcpy(to: display->wm.cur_latency, from: display->wm.pri_latency,
2883 len: sizeof(display->wm.pri_latency));
2884
2885 intel_fixup_spr_wm_latency(display, wm: display->wm.spr_latency);
2886 intel_fixup_cur_wm_latency(display, wm: display->wm.cur_latency);
2887
2888 intel_print_wm_latency(display, name: "Primary", wm: display->wm.pri_latency);
2889 intel_print_wm_latency(display, name: "Sprite", wm: display->wm.spr_latency);
2890 intel_print_wm_latency(display, name: "Cursor", wm: display->wm.cur_latency);
2891
2892 if (DISPLAY_VER(display) == 6) {
2893 snb_wm_latency_quirk(display);
2894 snb_wm_lp3_irq_quirk(display);
2895 }
2896}
2897
2898static bool ilk_validate_pipe_wm(struct intel_display *display,
2899 struct intel_pipe_wm *pipe_wm)
2900{
2901 /* LP0 watermark maximums depend on this pipe alone */
2902 const struct intel_wm_config config = {
2903 .num_pipes_active = 1,
2904 .sprites_enabled = pipe_wm->sprites_enabled,
2905 .sprites_scaled = pipe_wm->sprites_scaled,
2906 };
2907 struct ilk_wm_maximums max;
2908
2909 /* LP0 watermarks always use 1/2 DDB partitioning */
2910 ilk_compute_wm_maximums(display, level: 0, config: &config, ddb_partitioning: INTEL_DDB_PART_1_2, max: &max);
2911
2912 /* At least LP0 must be valid */
2913 if (!ilk_validate_wm_level(display, level: 0, max: &max, result: &pipe_wm->wm[0])) {
2914 drm_dbg_kms(display->drm, "LP0 watermark invalid\n");
2915 return false;
2916 }
2917
2918 return true;
2919}
2920
2921/* Compute new watermarks for the pipe */
2922static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2923 struct intel_crtc *crtc)
2924{
2925 struct intel_display *display = to_intel_display(state);
2926 struct intel_crtc_state *crtc_state =
2927 intel_atomic_get_new_crtc_state(state, crtc);
2928 struct intel_pipe_wm *pipe_wm;
2929 struct intel_plane *plane;
2930 const struct intel_plane_state *plane_state;
2931 const struct intel_plane_state *pristate = NULL;
2932 const struct intel_plane_state *sprstate = NULL;
2933 const struct intel_plane_state *curstate = NULL;
2934 struct ilk_wm_maximums max;
2935 int level, usable_level;
2936
2937 pipe_wm = &crtc_state->wm.ilk.optimal;
2938
2939 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2940 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2941 pristate = plane_state;
2942 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2943 sprstate = plane_state;
2944 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2945 curstate = plane_state;
2946 }
2947
2948 pipe_wm->pipe_enabled = crtc_state->hw.active;
2949 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2950 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2951
2952 usable_level = display->wm.num_levels - 1;
2953
2954 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2955 if (DISPLAY_VER(display) < 7 && pipe_wm->sprites_enabled)
2956 usable_level = 1;
2957
2958 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2959 if (pipe_wm->sprites_scaled)
2960 usable_level = 0;
2961
2962 memset(s: &pipe_wm->wm, c: 0, n: sizeof(pipe_wm->wm));
2963 ilk_compute_wm_level(display, crtc, level: 0, crtc_state,
2964 pristate, sprstate, curstate, result: &pipe_wm->wm[0]);
2965
2966 if (!ilk_validate_pipe_wm(display, pipe_wm))
2967 return -EINVAL;
2968
2969 ilk_compute_wm_reg_maximums(display, level: 1, max: &max);
2970
2971 for (level = 1; level <= usable_level; level++) {
2972 struct intel_wm_level *wm = &pipe_wm->wm[level];
2973
2974 ilk_compute_wm_level(display, crtc, level, crtc_state,
2975 pristate, sprstate, curstate, result: wm);
2976
2977 /*
2978 * Disable any watermark level that exceeds the
2979 * register maximums since such watermarks are
2980 * always invalid.
2981 */
2982 if (!ilk_validate_wm_level(display, level, max: &max, result: wm)) {
2983 memset(s: wm, c: 0, n: sizeof(*wm));
2984 break;
2985 }
2986 }
2987
2988 return 0;
2989}
2990
2991/*
2992 * Build a set of 'intermediate' watermark values that satisfy both the old
2993 * state and the new state. These can be programmed to the hardware
2994 * immediately.
2995 */
2996static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2997 struct intel_crtc *crtc)
2998{
2999 struct intel_display *display = to_intel_display(crtc);
3000 struct intel_crtc_state *new_crtc_state =
3001 intel_atomic_get_new_crtc_state(state, crtc);
3002 const struct intel_crtc_state *old_crtc_state =
3003 intel_atomic_get_old_crtc_state(state, crtc);
3004 struct intel_pipe_wm *intermediate = &new_crtc_state->wm.ilk.intermediate;
3005 const struct intel_pipe_wm *optimal = &new_crtc_state->wm.ilk.optimal;
3006 const struct intel_pipe_wm *active = &old_crtc_state->wm.ilk.optimal;
3007 int level;
3008
3009 /*
3010 * Start with the final, target watermarks, then combine with the
3011 * currently active watermarks to get values that are safe both before
3012 * and after the vblank.
3013 */
3014 *intermediate = *optimal;
3015 if (!new_crtc_state->hw.active ||
3016 intel_crtc_needs_modeset(crtc_state: new_crtc_state) ||
3017 state->skip_intermediate_wm)
3018 return 0;
3019
3020 intermediate->pipe_enabled |= active->pipe_enabled;
3021 intermediate->sprites_enabled |= active->sprites_enabled;
3022 intermediate->sprites_scaled |= active->sprites_scaled;
3023
3024 for (level = 0; level < display->wm.num_levels; level++) {
3025 struct intel_wm_level *intermediate_wm = &intermediate->wm[level];
3026 const struct intel_wm_level *active_wm = &active->wm[level];
3027
3028 intermediate_wm->enable &= active_wm->enable;
3029 intermediate_wm->pri_val = max(intermediate_wm->pri_val,
3030 active_wm->pri_val);
3031 intermediate_wm->spr_val = max(intermediate_wm->spr_val,
3032 active_wm->spr_val);
3033 intermediate_wm->cur_val = max(intermediate_wm->cur_val,
3034 active_wm->cur_val);
3035 intermediate_wm->fbc_val = max(intermediate_wm->fbc_val,
3036 active_wm->fbc_val);
3037 }
3038
3039 /*
3040 * We need to make sure that these merged watermark values are
3041 * actually a valid configuration themselves. If they're not,
3042 * there's no safe way to transition from the old state to
3043 * the new state, so we need to fail the atomic transaction.
3044 */
3045 if (!ilk_validate_pipe_wm(display, pipe_wm: intermediate))
3046 return -EINVAL;
3047
3048 /*
3049 * If our intermediate WM are identical to the final WM, then we can
3050 * omit the post-vblank programming; only update if it's different.
3051 */
3052 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
3053 new_crtc_state->wm.need_postvbl_update = true;
3054
3055 return 0;
3056}
3057
3058static int ilk_compute_watermarks(struct intel_atomic_state *state,
3059 struct intel_crtc *crtc)
3060{
3061 int ret;
3062
3063 ret = ilk_compute_pipe_wm(state, crtc);
3064 if (ret)
3065 return ret;
3066
3067 ret = ilk_compute_intermediate_wm(state, crtc);
3068 if (ret)
3069 return ret;
3070
3071 return 0;
3072}
3073
3074/*
3075 * Merge the watermarks from all active pipes for a specific level.
3076 */
3077static void ilk_merge_wm_level(struct intel_display *display,
3078 int level,
3079 struct intel_wm_level *ret_wm)
3080{
3081 const struct intel_crtc *crtc;
3082
3083 ret_wm->enable = true;
3084
3085 for_each_intel_crtc(display->drm, crtc) {
3086 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3087 const struct intel_wm_level *wm = &active->wm[level];
3088
3089 if (!active->pipe_enabled)
3090 continue;
3091
3092 /*
3093 * The watermark values may have been used in the past,
3094 * so we must maintain them in the registers for some
3095 * time even if the level is now disabled.
3096 */
3097 if (!wm->enable)
3098 ret_wm->enable = false;
3099
3100 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3101 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3102 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3103 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3104 }
3105}
3106
3107/*
3108 * Merge all low power watermarks for all active pipes.
3109 */
3110static void ilk_wm_merge(struct intel_display *display,
3111 const struct intel_wm_config *config,
3112 const struct ilk_wm_maximums *max,
3113 struct intel_pipe_wm *merged)
3114{
3115 int level, num_levels = display->wm.num_levels;
3116 int last_enabled_level = num_levels - 1;
3117
3118 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3119 if ((DISPLAY_VER(display) < 7 || display->platform.ivybridge) &&
3120 config->num_pipes_active > 1)
3121 last_enabled_level = 0;
3122
3123 /* ILK: FBC WM must be disabled always */
3124 merged->fbc_wm_enabled = DISPLAY_VER(display) >= 6;
3125
3126 /* merge each WM1+ level */
3127 for (level = 1; level < num_levels; level++) {
3128 struct intel_wm_level *wm = &merged->wm[level];
3129
3130 ilk_merge_wm_level(display, level, ret_wm: wm);
3131
3132 if (level > last_enabled_level)
3133 wm->enable = false;
3134 else if (!ilk_validate_wm_level(display, level, max, result: wm))
3135 /* make sure all following levels get disabled */
3136 last_enabled_level = level - 1;
3137
3138 /*
3139 * The spec says it is preferred to disable
3140 * FBC WMs instead of disabling a WM level.
3141 */
3142 if (wm->fbc_val > max->fbc) {
3143 if (wm->enable)
3144 merged->fbc_wm_enabled = false;
3145 wm->fbc_val = 0;
3146 }
3147 }
3148
3149 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3150 if (DISPLAY_VER(display) == 5 && HAS_FBC(display) &&
3151 display->params.enable_fbc && !merged->fbc_wm_enabled) {
3152 for (level = 2; level < num_levels; level++) {
3153 struct intel_wm_level *wm = &merged->wm[level];
3154
3155 wm->enable = false;
3156 }
3157 }
3158}
3159
3160static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3161{
3162 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3163 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3164}
3165
3166/* The value we need to program into the WM_LPx latency field */
3167static unsigned int ilk_wm_lp_latency(struct intel_display *display,
3168 int level)
3169{
3170 if (display->platform.haswell || display->platform.broadwell)
3171 return 2 * level;
3172 else
3173 return display->wm.pri_latency[level];
3174}
3175
3176static void ilk_compute_wm_results(struct intel_display *display,
3177 const struct intel_pipe_wm *merged,
3178 enum intel_ddb_partitioning partitioning,
3179 struct ilk_wm_values *results)
3180{
3181 struct intel_crtc *crtc;
3182 int level, wm_lp;
3183
3184 results->enable_fbc_wm = merged->fbc_wm_enabled;
3185 results->partitioning = partitioning;
3186
3187 /* LP1+ register values */
3188 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3189 const struct intel_wm_level *r;
3190
3191 level = ilk_wm_lp_to_level(wm_lp, pipe_wm: merged);
3192
3193 r = &merged->wm[level];
3194
3195 /*
3196 * Maintain the watermark values even if the level is
3197 * disabled. Doing otherwise could cause underruns.
3198 */
3199 results->wm_lp[wm_lp - 1] =
3200 WM_LP_LATENCY(ilk_wm_lp_latency(display, level)) |
3201 WM_LP_PRIMARY(r->pri_val) |
3202 WM_LP_CURSOR(r->cur_val);
3203
3204 if (r->enable)
3205 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3206
3207 if (DISPLAY_VER(display) >= 8)
3208 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3209 else
3210 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3211
3212 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3213
3214 /*
3215 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3216 * level is disabled. Doing otherwise could cause underruns.
3217 */
3218 if (DISPLAY_VER(display) < 7 && r->spr_val) {
3219 drm_WARN_ON(display->drm, wm_lp != 1);
3220 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3221 }
3222 }
3223
3224 /* LP0 register values */
3225 for_each_intel_crtc(display->drm, crtc) {
3226 enum pipe pipe = crtc->pipe;
3227 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3228 const struct intel_wm_level *r = &pipe_wm->wm[0];
3229
3230 if (drm_WARN_ON(display->drm, !r->enable))
3231 continue;
3232
3233 results->wm_pipe[pipe] =
3234 WM0_PIPE_PRIMARY(r->pri_val) |
3235 WM0_PIPE_SPRITE(r->spr_val) |
3236 WM0_PIPE_CURSOR(r->cur_val);
3237 }
3238}
3239
3240/*
3241 * Find the result with the highest level enabled. Check for enable_fbc_wm in
3242 * case both are at the same level. Prefer r1 in case they're the same.
3243 */
3244static struct intel_pipe_wm *
3245ilk_find_best_result(struct intel_display *display,
3246 struct intel_pipe_wm *r1,
3247 struct intel_pipe_wm *r2)
3248{
3249 int level, level1 = 0, level2 = 0;
3250
3251 for (level = 1; level < display->wm.num_levels; level++) {
3252 if (r1->wm[level].enable)
3253 level1 = level;
3254 if (r2->wm[level].enable)
3255 level2 = level;
3256 }
3257
3258 if (level1 == level2) {
3259 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3260 return r2;
3261 else
3262 return r1;
3263 } else if (level1 > level2) {
3264 return r1;
3265 } else {
3266 return r2;
3267 }
3268}
3269
3270/* dirty bits used to track which watermarks need changes */
3271#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3272#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3273#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3274#define WM_DIRTY_FBC (1 << 24)
3275#define WM_DIRTY_DDB (1 << 25)
3276
3277static unsigned int ilk_compute_wm_dirty(struct intel_display *display,
3278 const struct ilk_wm_values *old,
3279 const struct ilk_wm_values *new)
3280{
3281 unsigned int dirty = 0;
3282 enum pipe pipe;
3283 int wm_lp;
3284
3285 for_each_pipe(display, pipe) {
3286 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3287 dirty |= WM_DIRTY_PIPE(pipe);
3288 /* Must disable LP1+ watermarks too */
3289 dirty |= WM_DIRTY_LP_ALL;
3290 }
3291 }
3292
3293 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3294 dirty |= WM_DIRTY_FBC;
3295 /* Must disable LP1+ watermarks too */
3296 dirty |= WM_DIRTY_LP_ALL;
3297 }
3298
3299 if (old->partitioning != new->partitioning) {
3300 dirty |= WM_DIRTY_DDB;
3301 /* Must disable LP1+ watermarks too */
3302 dirty |= WM_DIRTY_LP_ALL;
3303 }
3304
3305 /* LP1+ watermarks already deemed dirty, no need to continue */
3306 if (dirty & WM_DIRTY_LP_ALL)
3307 return dirty;
3308
3309 /* Find the lowest numbered LP1+ watermark in need of an update... */
3310 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3311 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3312 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3313 break;
3314 }
3315
3316 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3317 for (; wm_lp <= 3; wm_lp++)
3318 dirty |= WM_DIRTY_LP(wm_lp);
3319
3320 return dirty;
3321}
3322
3323static bool _ilk_disable_lp_wm(struct intel_display *display,
3324 unsigned int dirty)
3325{
3326 struct ilk_wm_values *previous = &display->wm.hw;
3327 bool changed = false;
3328
3329 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3330 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3331 intel_de_write(display, WM3_LP_ILK, val: previous->wm_lp[2]);
3332 changed = true;
3333 }
3334 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3335 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3336 intel_de_write(display, WM2_LP_ILK, val: previous->wm_lp[1]);
3337 changed = true;
3338 }
3339 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3340 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3341 intel_de_write(display, WM1_LP_ILK, val: previous->wm_lp[0]);
3342 changed = true;
3343 }
3344
3345 /*
3346 * Don't touch WM_LP_SPRITE_ENABLE here.
3347 * Doing so could cause underruns.
3348 */
3349
3350 return changed;
3351}
3352
3353/*
3354 * The spec says we shouldn't write when we don't need, because every write
3355 * causes WMs to be re-evaluated, expending some power.
3356 */
3357static void ilk_write_wm_values(struct intel_display *display,
3358 struct ilk_wm_values *results)
3359{
3360 struct ilk_wm_values *previous = &display->wm.hw;
3361 unsigned int dirty;
3362
3363 dirty = ilk_compute_wm_dirty(display, old: previous, new: results);
3364 if (!dirty)
3365 return;
3366
3367 _ilk_disable_lp_wm(display, dirty);
3368
3369 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3370 intel_de_write(display, WM0_PIPE_ILK(PIPE_A), val: results->wm_pipe[0]);
3371 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3372 intel_de_write(display, WM0_PIPE_ILK(PIPE_B), val: results->wm_pipe[1]);
3373 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3374 intel_de_write(display, WM0_PIPE_ILK(PIPE_C), val: results->wm_pipe[2]);
3375
3376 if (dirty & WM_DIRTY_DDB) {
3377 if (display->platform.haswell || display->platform.broadwell)
3378 intel_de_rmw(display, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3379 set: results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3380 WM_MISC_DATA_PARTITION_5_6);
3381 else
3382 intel_de_rmw(display, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3383 set: results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3384 DISP_DATA_PARTITION_5_6);
3385 }
3386
3387 if (dirty & WM_DIRTY_FBC)
3388 intel_de_rmw(display, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3389 set: results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3390
3391 if (dirty & WM_DIRTY_LP(1) &&
3392 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3393 intel_de_write(display, WM1S_LP_ILK, val: results->wm_lp_spr[0]);
3394
3395 if (DISPLAY_VER(display) >= 7) {
3396 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3397 intel_de_write(display, WM2S_LP_IVB, val: results->wm_lp_spr[1]);
3398 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3399 intel_de_write(display, WM3S_LP_IVB, val: results->wm_lp_spr[2]);
3400 }
3401
3402 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3403 intel_de_write(display, WM1_LP_ILK, val: results->wm_lp[0]);
3404 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3405 intel_de_write(display, WM2_LP_ILK, val: results->wm_lp[1]);
3406 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3407 intel_de_write(display, WM3_LP_ILK, val: results->wm_lp[2]);
3408
3409 display->wm.hw = *results;
3410}
3411
3412bool ilk_disable_cxsr(struct intel_display *display)
3413{
3414 return _ilk_disable_lp_wm(display, WM_DIRTY_LP_ALL);
3415}
3416
3417static void ilk_compute_wm_config(struct intel_display *display,
3418 struct intel_wm_config *config)
3419{
3420 struct intel_crtc *crtc;
3421
3422 /* Compute the currently _active_ config */
3423 for_each_intel_crtc(display->drm, crtc) {
3424 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3425
3426 if (!wm->pipe_enabled)
3427 continue;
3428
3429 config->sprites_enabled |= wm->sprites_enabled;
3430 config->sprites_scaled |= wm->sprites_scaled;
3431 config->num_pipes_active++;
3432 }
3433}
3434
3435static void ilk_program_watermarks(struct intel_display *display)
3436{
3437 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3438 struct ilk_wm_maximums max;
3439 struct intel_wm_config config = {};
3440 struct ilk_wm_values results = {};
3441 enum intel_ddb_partitioning partitioning;
3442
3443 ilk_compute_wm_config(display, config: &config);
3444
3445 ilk_compute_wm_maximums(display, level: 1, config: &config, ddb_partitioning: INTEL_DDB_PART_1_2, max: &max);
3446 ilk_wm_merge(display, config: &config, max: &max, merged: &lp_wm_1_2);
3447
3448 /* 5/6 split only in single pipe config on IVB+ */
3449 if (DISPLAY_VER(display) >= 7 &&
3450 config.num_pipes_active == 1 && config.sprites_enabled) {
3451 ilk_compute_wm_maximums(display, level: 1, config: &config, ddb_partitioning: INTEL_DDB_PART_5_6, max: &max);
3452 ilk_wm_merge(display, config: &config, max: &max, merged: &lp_wm_5_6);
3453
3454 best_lp_wm = ilk_find_best_result(display, r1: &lp_wm_1_2, r2: &lp_wm_5_6);
3455 } else {
3456 best_lp_wm = &lp_wm_1_2;
3457 }
3458
3459 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3460 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3461
3462 ilk_compute_wm_results(display, merged: best_lp_wm, partitioning, results: &results);
3463
3464 ilk_write_wm_values(display, results: &results);
3465}
3466
3467static void ilk_initial_watermarks(struct intel_atomic_state *state,
3468 struct intel_crtc *crtc)
3469{
3470 struct intel_display *display = to_intel_display(crtc);
3471 const struct intel_crtc_state *crtc_state =
3472 intel_atomic_get_new_crtc_state(state, crtc);
3473
3474 mutex_lock(lock: &display->wm.wm_mutex);
3475 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3476 ilk_program_watermarks(display);
3477 mutex_unlock(lock: &display->wm.wm_mutex);
3478}
3479
3480static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3481 struct intel_crtc *crtc)
3482{
3483 struct intel_display *display = to_intel_display(crtc);
3484 const struct intel_crtc_state *crtc_state =
3485 intel_atomic_get_new_crtc_state(state, crtc);
3486
3487 if (!crtc_state->wm.need_postvbl_update)
3488 return;
3489
3490 mutex_lock(lock: &display->wm.wm_mutex);
3491 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3492 ilk_program_watermarks(display);
3493 mutex_unlock(lock: &display->wm.wm_mutex);
3494}
3495
3496static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3497{
3498 struct intel_display *display = to_intel_display(crtc);
3499 struct ilk_wm_values *hw = &display->wm.hw;
3500 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3501 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3502 enum pipe pipe = crtc->pipe;
3503
3504 hw->wm_pipe[pipe] = intel_de_read(display, WM0_PIPE_ILK(pipe));
3505
3506 memset(s: active, c: 0, n: sizeof(*active));
3507
3508 active->pipe_enabled = crtc->active;
3509
3510 if (active->pipe_enabled) {
3511 u32 tmp = hw->wm_pipe[pipe];
3512
3513 /*
3514 * For active pipes LP0 watermark is marked as
3515 * enabled, and LP1+ watermaks as disabled since
3516 * we can't really reverse compute them in case
3517 * multiple pipes are active.
3518 */
3519 active->wm[0].enable = true;
3520 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3521 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3522 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3523 } else {
3524 int level;
3525
3526 /*
3527 * For inactive pipes, all watermark levels
3528 * should be marked as enabled but zeroed,
3529 * which is what we'd compute them to.
3530 */
3531 for (level = 0; level < display->wm.num_levels; level++)
3532 active->wm[level].enable = true;
3533 }
3534
3535 crtc->wm.active.ilk = *active;
3536}
3537
3538static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3539{
3540 struct drm_plane *plane;
3541 struct intel_crtc *crtc;
3542
3543 for_each_intel_crtc(state->dev, crtc) {
3544 struct intel_crtc_state *crtc_state;
3545
3546 crtc_state = intel_atomic_get_crtc_state(state, crtc);
3547 if (IS_ERR(ptr: crtc_state))
3548 return PTR_ERR(ptr: crtc_state);
3549
3550 if (crtc_state->hw.active) {
3551 /*
3552 * Preserve the inherited flag to avoid
3553 * taking the full modeset path.
3554 */
3555 crtc_state->inherited = true;
3556 }
3557 }
3558
3559 drm_for_each_plane(plane, state->dev) {
3560 struct drm_plane_state *plane_state;
3561
3562 plane_state = drm_atomic_get_plane_state(state, plane);
3563 if (IS_ERR(ptr: plane_state))
3564 return PTR_ERR(ptr: plane_state);
3565 }
3566
3567 return 0;
3568}
3569
3570/*
3571 * Calculate what we think the watermarks should be for the state we've read
3572 * out of the hardware and then immediately program those watermarks so that
3573 * we ensure the hardware settings match our internal state.
3574 *
3575 * We can calculate what we think WM's should be by creating a duplicate of the
3576 * current state (which was constructed during hardware readout) and running it
3577 * through the atomic check code to calculate new watermark values in the
3578 * state object.
3579 */
3580void ilk_wm_sanitize(struct intel_display *display)
3581{
3582 struct drm_atomic_state *state;
3583 struct intel_atomic_state *intel_state;
3584 struct intel_crtc *crtc;
3585 struct intel_crtc_state *crtc_state;
3586 struct drm_modeset_acquire_ctx ctx;
3587 int ret;
3588 int i;
3589
3590 /* Only supported on platforms that use atomic watermark design */
3591 if (!display->funcs.wm->optimize_watermarks)
3592 return;
3593
3594 if (drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 9))
3595 return;
3596
3597 state = drm_atomic_state_alloc(dev: display->drm);
3598 if (drm_WARN_ON(display->drm, !state))
3599 return;
3600
3601 intel_state = to_intel_atomic_state(state);
3602
3603 drm_modeset_acquire_init(ctx: &ctx, flags: 0);
3604
3605 state->acquire_ctx = &ctx;
3606 to_intel_atomic_state(state)->internal = true;
3607
3608retry:
3609 /*
3610 * Hardware readout is the only time we don't want to calculate
3611 * intermediate watermarks (since we don't trust the current
3612 * watermarks).
3613 */
3614 if (!HAS_GMCH(display))
3615 intel_state->skip_intermediate_wm = true;
3616
3617 ret = ilk_sanitize_watermarks_add_affected(state);
3618 if (ret)
3619 goto fail;
3620
3621 ret = intel_atomic_check(dev: display->drm, state);
3622 if (ret)
3623 goto fail;
3624
3625 /* Write calculated watermark values back */
3626 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3627 crtc_state->wm.need_postvbl_update = true;
3628 intel_optimize_watermarks(state: intel_state, crtc);
3629
3630 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3631 }
3632
3633fail:
3634 if (ret == -EDEADLK) {
3635 drm_atomic_state_clear(state);
3636 drm_modeset_backoff(ctx: &ctx);
3637 goto retry;
3638 }
3639
3640 /*
3641 * If we fail here, it means that the hardware appears to be
3642 * programmed in a way that shouldn't be possible, given our
3643 * understanding of watermark requirements. This might mean a
3644 * mistake in the hardware readout code or a mistake in the
3645 * watermark calculations for a given platform. Raise a WARN
3646 * so that this is noticeable.
3647 *
3648 * If this actually happens, we'll have to just leave the
3649 * BIOS-programmed watermarks untouched and hope for the best.
3650 */
3651 drm_WARN(display->drm, ret,
3652 "Could not determine valid watermarks for inherited state\n");
3653
3654 drm_atomic_state_put(state);
3655
3656 drm_modeset_drop_locks(ctx: &ctx);
3657 drm_modeset_acquire_fini(ctx: &ctx);
3658}
3659
3660#define _FW_WM(value, plane) \
3661 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3662#define _FW_WM_VLV(value, plane) \
3663 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3664
3665static void g4x_read_wm_values(struct intel_display *display,
3666 struct g4x_wm_values *wm)
3667{
3668 u32 tmp;
3669
3670 tmp = intel_de_read(display, DSPFW1(display));
3671 wm->sr.plane = _FW_WM(tmp, SR);
3672 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3673 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3674 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3675
3676 tmp = intel_de_read(display, DSPFW2(display));
3677 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3678 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3679 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3680 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3681 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3682 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3683
3684 tmp = intel_de_read(display, DSPFW3(display));
3685 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3686 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3687 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3688 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3689}
3690
3691static void vlv_read_wm_values(struct intel_display *display,
3692 struct vlv_wm_values *wm)
3693{
3694 enum pipe pipe;
3695 u32 tmp;
3696
3697 for_each_pipe(display, pipe) {
3698 tmp = intel_de_read(display, VLV_DDL(pipe));
3699
3700 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3701 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3702 wm->ddl[pipe].plane[PLANE_CURSOR] =
3703 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3704 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3705 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3706 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3707 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3708 }
3709
3710 tmp = intel_de_read(display, DSPFW1(display));
3711 wm->sr.plane = _FW_WM(tmp, SR);
3712 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3713 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3714 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3715
3716 tmp = intel_de_read(display, DSPFW2(display));
3717 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3718 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3719 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3720
3721 tmp = intel_de_read(display, DSPFW3(display));
3722 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3723
3724 if (display->platform.cherryview) {
3725 tmp = intel_de_read(display, DSPFW7_CHV);
3726 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3727 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3728
3729 tmp = intel_de_read(display, DSPFW8_CHV);
3730 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3731 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3732
3733 tmp = intel_de_read(display, DSPFW9_CHV);
3734 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3735 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3736
3737 tmp = intel_de_read(display, DSPHOWM);
3738 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3739 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3740 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3741 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3742 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3743 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3744 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3745 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3746 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3747 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3748 } else {
3749 tmp = intel_de_read(display, DSPFW7);
3750 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3751 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3752
3753 tmp = intel_de_read(display, DSPHOWM);
3754 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3755 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3756 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3757 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3758 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3759 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3760 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3761 }
3762}
3763
3764#undef _FW_WM
3765#undef _FW_WM_VLV
3766
3767static void g4x_wm_get_hw_state(struct intel_display *display)
3768{
3769 struct g4x_wm_values *wm = &display->wm.g4x;
3770 struct intel_crtc *crtc;
3771
3772 g4x_read_wm_values(display, wm);
3773
3774 wm->cxsr = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
3775
3776 for_each_intel_crtc(display->drm, crtc) {
3777 struct intel_crtc_state *crtc_state =
3778 to_intel_crtc_state(crtc->base.state);
3779 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3780 struct g4x_pipe_wm *raw;
3781 enum pipe pipe = crtc->pipe;
3782 enum plane_id plane_id;
3783 int level, max_level;
3784
3785 active->cxsr = wm->cxsr;
3786 active->hpll_en = wm->hpll_en;
3787 active->fbc_en = wm->fbc_en;
3788
3789 active->sr = wm->sr;
3790 active->hpll = wm->hpll;
3791
3792 for_each_plane_id_on_crtc(crtc, plane_id) {
3793 active->wm.plane[plane_id] =
3794 wm->pipe[pipe].plane[plane_id];
3795 }
3796
3797 if (wm->cxsr && wm->hpll_en)
3798 max_level = G4X_WM_LEVEL_HPLL;
3799 else if (wm->cxsr)
3800 max_level = G4X_WM_LEVEL_SR;
3801 else
3802 max_level = G4X_WM_LEVEL_NORMAL;
3803
3804 level = G4X_WM_LEVEL_NORMAL;
3805 raw = &crtc_state->wm.g4x.raw[level];
3806 for_each_plane_id_on_crtc(crtc, plane_id)
3807 raw->plane[plane_id] = active->wm.plane[plane_id];
3808
3809 level = G4X_WM_LEVEL_SR;
3810 if (level > max_level)
3811 goto out;
3812
3813 raw = &crtc_state->wm.g4x.raw[level];
3814 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3815 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3816 raw->plane[PLANE_SPRITE0] = 0;
3817 raw->fbc = active->sr.fbc;
3818
3819 level = G4X_WM_LEVEL_HPLL;
3820 if (level > max_level)
3821 goto out;
3822
3823 raw = &crtc_state->wm.g4x.raw[level];
3824 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3825 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3826 raw->plane[PLANE_SPRITE0] = 0;
3827 raw->fbc = active->hpll.fbc;
3828
3829 level++;
3830 out:
3831 for_each_plane_id_on_crtc(crtc, plane_id)
3832 g4x_raw_plane_wm_set(crtc_state, level,
3833 plane_id, USHRT_MAX);
3834 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3835
3836 g4x_invalidate_wms(crtc, wm_state: active, level);
3837
3838 crtc_state->wm.g4x.optimal = *active;
3839 crtc_state->wm.g4x.intermediate = *active;
3840
3841 drm_dbg_kms(display->drm,
3842 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3843 pipe_name(pipe),
3844 wm->pipe[pipe].plane[PLANE_PRIMARY],
3845 wm->pipe[pipe].plane[PLANE_CURSOR],
3846 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3847 }
3848
3849 drm_dbg_kms(display->drm,
3850 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3851 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3852 drm_dbg_kms(display->drm,
3853 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3854 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3855 drm_dbg_kms(display->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3856 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3857 str_yes_no(wm->fbc_en));
3858}
3859
3860static void g4x_wm_sanitize(struct intel_display *display)
3861{
3862 struct intel_plane *plane;
3863 struct intel_crtc *crtc;
3864
3865 mutex_lock(lock: &display->wm.wm_mutex);
3866
3867 for_each_intel_plane(display->drm, plane) {
3868 struct intel_crtc *crtc =
3869 intel_crtc_for_pipe(display, pipe: plane->pipe);
3870 struct intel_crtc_state *crtc_state =
3871 to_intel_crtc_state(crtc->base.state);
3872 struct intel_plane_state *plane_state =
3873 to_intel_plane_state(plane->base.state);
3874 enum plane_id plane_id = plane->id;
3875 int level;
3876
3877 if (plane_state->uapi.visible)
3878 continue;
3879
3880 for (level = 0; level < display->wm.num_levels; level++) {
3881 struct g4x_pipe_wm *raw =
3882 &crtc_state->wm.g4x.raw[level];
3883
3884 raw->plane[plane_id] = 0;
3885
3886 if (plane_id == PLANE_PRIMARY)
3887 raw->fbc = 0;
3888 }
3889 }
3890
3891 for_each_intel_crtc(display->drm, crtc) {
3892 struct intel_crtc_state *crtc_state =
3893 to_intel_crtc_state(crtc->base.state);
3894 int ret;
3895
3896 ret = _g4x_compute_pipe_wm(crtc_state);
3897 drm_WARN_ON(display->drm, ret);
3898
3899 crtc_state->wm.g4x.intermediate =
3900 crtc_state->wm.g4x.optimal;
3901 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3902 }
3903
3904 g4x_program_watermarks(display);
3905
3906 mutex_unlock(lock: &display->wm.wm_mutex);
3907}
3908
3909static void vlv_wm_get_hw_state(struct intel_display *display)
3910{
3911 struct vlv_wm_values *wm = &display->wm.vlv;
3912 struct intel_crtc *crtc;
3913 u32 val;
3914 int ret;
3915
3916 vlv_read_wm_values(display, wm);
3917
3918 wm->cxsr = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3919 wm->level = VLV_WM_LEVEL_PM2;
3920
3921 if (display->platform.cherryview) {
3922 vlv_punit_get(drm: display->drm);
3923
3924 val = vlv_punit_read(drm: display->drm, PUNIT_REG_DSPSSPM);
3925 if (val & DSP_MAXFIFO_PM5_ENABLE)
3926 wm->level = VLV_WM_LEVEL_PM5;
3927
3928 /*
3929 * If DDR DVFS is disabled in the BIOS, Punit
3930 * will never ack the request. So if that happens
3931 * assume we don't have to enable/disable DDR DVFS
3932 * dynamically. To test that just set the REQ_ACK
3933 * bit to poke the Punit, but don't change the
3934 * HIGH/LOW bits so that we don't actually change
3935 * the current state.
3936 */
3937 val = vlv_punit_read(drm: display->drm, PUNIT_REG_DDR_SETUP2);
3938 val |= FORCE_DDR_FREQ_REQ_ACK;
3939 vlv_punit_write(drm: display->drm, PUNIT_REG_DDR_SETUP2, val);
3940
3941 ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
3942 (val & FORCE_DDR_FREQ_REQ_ACK) == 0,
3943 500, 3000, false);
3944 if (ret) {
3945 drm_dbg_kms(display->drm,
3946 "Punit not acking DDR DVFS request, "
3947 "assuming DDR DVFS is disabled\n");
3948 display->wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3949 } else {
3950 val = vlv_punit_read(drm: display->drm, PUNIT_REG_DDR_SETUP2);
3951 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3952 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3953 }
3954
3955 vlv_punit_put(drm: display->drm);
3956 }
3957
3958 for_each_intel_crtc(display->drm, crtc) {
3959 struct intel_crtc_state *crtc_state =
3960 to_intel_crtc_state(crtc->base.state);
3961 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3962 const struct vlv_fifo_state *fifo_state =
3963 &crtc_state->wm.vlv.fifo_state;
3964 enum pipe pipe = crtc->pipe;
3965 enum plane_id plane_id;
3966 int level;
3967
3968 vlv_get_fifo_size(crtc_state);
3969
3970 active->num_levels = wm->level + 1;
3971 active->cxsr = wm->cxsr;
3972
3973 for (level = 0; level < active->num_levels; level++) {
3974 struct g4x_pipe_wm *raw =
3975 &crtc_state->wm.vlv.raw[level];
3976
3977 active->sr[level].plane = wm->sr.plane;
3978 active->sr[level].cursor = wm->sr.cursor;
3979
3980 for_each_plane_id_on_crtc(crtc, plane_id) {
3981 active->wm[level].plane[plane_id] =
3982 wm->pipe[pipe].plane[plane_id];
3983
3984 raw->plane[plane_id] =
3985 vlv_invert_wm_value(wm: active->wm[level].plane[plane_id],
3986 fifo_size: fifo_state->plane[plane_id]);
3987 }
3988 }
3989
3990 for_each_plane_id_on_crtc(crtc, plane_id)
3991 vlv_raw_plane_wm_set(crtc_state, level,
3992 plane_id, USHRT_MAX);
3993 vlv_invalidate_wms(crtc, wm_state: active, level);
3994
3995 crtc_state->wm.vlv.optimal = *active;
3996 crtc_state->wm.vlv.intermediate = *active;
3997
3998 drm_dbg_kms(display->drm,
3999 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
4000 pipe_name(pipe),
4001 wm->pipe[pipe].plane[PLANE_PRIMARY],
4002 wm->pipe[pipe].plane[PLANE_CURSOR],
4003 wm->pipe[pipe].plane[PLANE_SPRITE0],
4004 wm->pipe[pipe].plane[PLANE_SPRITE1]);
4005 }
4006
4007 drm_dbg_kms(display->drm,
4008 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4009 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4010}
4011
4012static void vlv_wm_sanitize(struct intel_display *display)
4013{
4014 struct intel_plane *plane;
4015 struct intel_crtc *crtc;
4016
4017 mutex_lock(lock: &display->wm.wm_mutex);
4018
4019 for_each_intel_plane(display->drm, plane) {
4020 struct intel_crtc *crtc =
4021 intel_crtc_for_pipe(display, pipe: plane->pipe);
4022 struct intel_crtc_state *crtc_state =
4023 to_intel_crtc_state(crtc->base.state);
4024 struct intel_plane_state *plane_state =
4025 to_intel_plane_state(plane->base.state);
4026 enum plane_id plane_id = plane->id;
4027 int level;
4028
4029 if (plane_state->uapi.visible)
4030 continue;
4031
4032 for (level = 0; level < display->wm.num_levels; level++) {
4033 struct g4x_pipe_wm *raw =
4034 &crtc_state->wm.vlv.raw[level];
4035
4036 raw->plane[plane_id] = 0;
4037 }
4038 }
4039
4040 for_each_intel_crtc(display->drm, crtc) {
4041 struct intel_crtc_state *crtc_state =
4042 to_intel_crtc_state(crtc->base.state);
4043 int ret;
4044
4045 ret = _vlv_compute_pipe_wm(crtc_state);
4046 drm_WARN_ON(display->drm, ret);
4047
4048 crtc_state->wm.vlv.intermediate =
4049 crtc_state->wm.vlv.optimal;
4050 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4051 }
4052
4053 vlv_program_watermarks(display);
4054
4055 mutex_unlock(lock: &display->wm.wm_mutex);
4056}
4057
4058/*
4059 * FIXME should probably kill this and improve
4060 * the real watermark readout/sanitation instead
4061 */
4062static void ilk_init_lp_watermarks(struct intel_display *display)
4063{
4064 intel_de_rmw(display, WM3_LP_ILK, WM_LP_ENABLE, set: 0);
4065 intel_de_rmw(display, WM2_LP_ILK, WM_LP_ENABLE, set: 0);
4066 intel_de_rmw(display, WM1_LP_ILK, WM_LP_ENABLE, set: 0);
4067
4068 /*
4069 * Don't touch WM_LP_SPRITE_ENABLE here.
4070 * Doing so could cause underruns.
4071 */
4072}
4073
4074static void ilk_wm_get_hw_state(struct intel_display *display)
4075{
4076 struct ilk_wm_values *hw = &display->wm.hw;
4077 struct intel_crtc *crtc;
4078
4079 ilk_init_lp_watermarks(display);
4080
4081 for_each_intel_crtc(display->drm, crtc)
4082 ilk_pipe_wm_get_hw_state(crtc);
4083
4084 hw->wm_lp[0] = intel_de_read(display, WM1_LP_ILK);
4085 hw->wm_lp[1] = intel_de_read(display, WM2_LP_ILK);
4086 hw->wm_lp[2] = intel_de_read(display, WM3_LP_ILK);
4087
4088 hw->wm_lp_spr[0] = intel_de_read(display, WM1S_LP_ILK);
4089 if (DISPLAY_VER(display) >= 7) {
4090 hw->wm_lp_spr[1] = intel_de_read(display, WM2S_LP_IVB);
4091 hw->wm_lp_spr[2] = intel_de_read(display, WM3S_LP_IVB);
4092 }
4093
4094 if (display->platform.haswell || display->platform.broadwell)
4095 hw->partitioning = (intel_de_read(display, WM_MISC) &
4096 WM_MISC_DATA_PARTITION_5_6) ?
4097 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4098 else if (display->platform.ivybridge)
4099 hw->partitioning = (intel_de_read(display, DISP_ARB_CTL2) &
4100 DISP_DATA_PARTITION_5_6) ?
4101 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4102
4103 hw->enable_fbc_wm =
4104 !(intel_de_read(display, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4105}
4106
4107static const struct intel_wm_funcs ilk_wm_funcs = {
4108 .compute_watermarks = ilk_compute_watermarks,
4109 .initial_watermarks = ilk_initial_watermarks,
4110 .optimize_watermarks = ilk_optimize_watermarks,
4111 .get_hw_state = ilk_wm_get_hw_state,
4112};
4113
4114static const struct intel_wm_funcs vlv_wm_funcs = {
4115 .compute_watermarks = vlv_compute_watermarks,
4116 .initial_watermarks = vlv_initial_watermarks,
4117 .optimize_watermarks = vlv_optimize_watermarks,
4118 .atomic_update_watermarks = vlv_atomic_update_fifo,
4119 .get_hw_state = vlv_wm_get_hw_state,
4120 .sanitize = vlv_wm_sanitize,
4121};
4122
4123static const struct intel_wm_funcs g4x_wm_funcs = {
4124 .compute_watermarks = g4x_compute_watermarks,
4125 .initial_watermarks = g4x_initial_watermarks,
4126 .optimize_watermarks = g4x_optimize_watermarks,
4127 .get_hw_state = g4x_wm_get_hw_state,
4128 .sanitize = g4x_wm_sanitize,
4129};
4130
4131static const struct intel_wm_funcs pnv_wm_funcs = {
4132 .compute_watermarks = i9xx_compute_watermarks,
4133 .update_wm = pnv_update_wm,
4134};
4135
4136static const struct intel_wm_funcs i965_wm_funcs = {
4137 .compute_watermarks = i9xx_compute_watermarks,
4138 .update_wm = i965_update_wm,
4139};
4140
4141static const struct intel_wm_funcs i9xx_wm_funcs = {
4142 .compute_watermarks = i9xx_compute_watermarks,
4143 .update_wm = i9xx_update_wm,
4144};
4145
4146static const struct intel_wm_funcs i845_wm_funcs = {
4147 .compute_watermarks = i9xx_compute_watermarks,
4148 .update_wm = i845_update_wm,
4149};
4150
4151static const struct intel_wm_funcs nop_funcs = {
4152};
4153
4154void i9xx_wm_init(struct intel_display *display)
4155{
4156 /* For FIFO watermark updates */
4157 if (HAS_PCH_SPLIT(display)) {
4158 ilk_setup_wm_latency(display);
4159 display->funcs.wm = &ilk_wm_funcs;
4160 } else if (display->platform.valleyview || display->platform.cherryview) {
4161 vlv_setup_wm_latency(display);
4162 display->funcs.wm = &vlv_wm_funcs;
4163 } else if (display->platform.g4x) {
4164 g4x_setup_wm_latency(display);
4165 display->funcs.wm = &g4x_wm_funcs;
4166 } else if (display->platform.pineview) {
4167 if (!pnv_get_cxsr_latency(display)) {
4168 drm_info(display->drm, "Unknown FSB/MEM, disabling CxSR\n");
4169 /* Disable CxSR and never update its watermark again */
4170 intel_set_memory_cxsr(display, enable: false);
4171 display->funcs.wm = &nop_funcs;
4172 } else {
4173 display->funcs.wm = &pnv_wm_funcs;
4174 }
4175 } else if (DISPLAY_VER(display) == 4) {
4176 display->funcs.wm = &i965_wm_funcs;
4177 } else if (DISPLAY_VER(display) == 3) {
4178 display->funcs.wm = &i9xx_wm_funcs;
4179 } else if (DISPLAY_VER(display) == 2) {
4180 if (INTEL_NUM_PIPES(display) == 1)
4181 display->funcs.wm = &i845_wm_funcs;
4182 else
4183 display->funcs.wm = &i9xx_wm_funcs;
4184 } else {
4185 drm_err(display->drm,
4186 "unexpected fall-through in %s\n", __func__);
4187 display->funcs.wm = &nop_funcs;
4188 }
4189}
4190