1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include "display/i9xx_plane_regs.h"
29#include "display/intel_display.h"
30#include "display/intel_display_core.h"
31
32#include "gt/intel_engine_regs.h"
33#include "gt/intel_gt.h"
34#include "gt/intel_gt_mcr.h"
35#include "gt/intel_gt_regs.h"
36
37#include "i915_drv.h"
38#include "i915_reg.h"
39#include "intel_clock_gating.h"
40#include "intel_mchbar_regs.h"
41#include "vlv_iosf_sb.h"
42
43struct drm_i915_clock_gating_funcs {
44 void (*init_clock_gating)(struct drm_i915_private *i915);
45};
46
47static void gen9_init_clock_gating(struct drm_i915_private *i915)
48{
49 if (HAS_LLC(i915)) {
50 /*
51 * WaCompressedResourceDisplayNewHashMode:skl,kbl
52 * Display WA #0390: skl,kbl
53 *
54 * Must match Sampler, Pixel Back End, and Media. See
55 * WaCompressedResourceSamplerPbeMediaNewHashMode.
56 */
57 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PAR1_1, clear: 0, SKL_DE_COMPRESSED_HASH_MODE);
58 }
59
60 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
61 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PAR1_1, clear: 0, SKL_EDP_PSR_FIX_RDWRAP);
62
63 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
64 intel_uncore_rmw(uncore: &i915->uncore, GEN8_CHICKEN_DCPR_1, clear: 0, MASK_WAKEMEM);
65
66 /*
67 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
68 * Display WA #0859: skl,bxt,kbl,glk,cfl
69 */
70 intel_uncore_rmw(uncore: &i915->uncore, DISP_ARB_CTL, clear: 0, DISP_FBC_MEMORY_WAKE);
71}
72
73static void bxt_init_clock_gating(struct drm_i915_private *i915)
74{
75 gen9_init_clock_gating(i915);
76
77 /* WaDisableSDEUnitClockGating:bxt */
78 intel_uncore_rmw(uncore: &i915->uncore, GEN8_UCGCTL6, clear: 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
79
80 /*
81 * FIXME:
82 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
83 */
84 intel_uncore_rmw(uncore: &i915->uncore, GEN8_UCGCTL6, clear: 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
85
86 /*
87 * Wa: Backlight PWM may stop in the asserted state, causing backlight
88 * to stay fully on.
89 */
90 intel_uncore_write(uncore: &i915->uncore, GEN9_CLKGATE_DIS_0,
91 val: intel_uncore_read(uncore: &i915->uncore, GEN9_CLKGATE_DIS_0) |
92 PWM1_GATING_DIS | PWM2_GATING_DIS);
93
94 /*
95 * Lower the display internal timeout.
96 * This is needed to avoid any hard hangs when DSI port PLL
97 * is off and a MMIO access is attempted by any privilege
98 * application, using batch buffers or any other means.
99 */
100 intel_uncore_write(uncore: &i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
101
102 /*
103 * WaFbcTurnOffFbcWatermark:bxt
104 * Display WA #0562: bxt
105 */
106 intel_uncore_rmw(uncore: &i915->uncore, DISP_ARB_CTL, clear: 0, DISP_FBC_WM_DIS);
107}
108
109static void glk_init_clock_gating(struct drm_i915_private *i915)
110{
111 gen9_init_clock_gating(i915);
112
113 /*
114 * WaDisablePWMClockGating:glk
115 * Backlight PWM may stop in the asserted state, causing backlight
116 * to stay fully on.
117 */
118 intel_uncore_write(uncore: &i915->uncore, GEN9_CLKGATE_DIS_0,
119 val: intel_uncore_read(uncore: &i915->uncore, GEN9_CLKGATE_DIS_0) |
120 PWM1_GATING_DIS | PWM2_GATING_DIS);
121}
122
123static void ibx_init_clock_gating(struct drm_i915_private *i915)
124{
125 /*
126 * On Ibex Peak and Cougar Point, we need to disable clock
127 * gating for the panel power sequencer or it will fail to
128 * start up when no ports are active.
129 */
130 intel_uncore_write(uncore: &i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
131}
132
133static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
134{
135 struct intel_display *display = dev_priv->display;
136 enum pipe pipe;
137
138 for_each_pipe(display, pipe) {
139 intel_uncore_rmw(uncore: &dev_priv->uncore, DSPCNTR(display, pipe),
140 clear: 0, DISP_TRICKLE_FEED_DISABLE);
141
142 intel_uncore_rmw(uncore: &dev_priv->uncore, DSPSURF(display, pipe),
143 clear: 0, set: 0);
144 intel_uncore_posting_read(&dev_priv->uncore,
145 DSPSURF(display, pipe));
146 }
147}
148
149static void ilk_init_clock_gating(struct drm_i915_private *i915)
150{
151 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
152
153 /*
154 * Required for FBC
155 * WaFbcDisableDpfcClockGating:ilk
156 */
157 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
158 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
159 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
160
161 intel_uncore_write(uncore: &i915->uncore, PCH_3DCGDIS0,
162 MARIUNIT_CLOCK_GATE_DISABLE |
163 SVSMUNIT_CLOCK_GATE_DISABLE);
164 intel_uncore_write(uncore: &i915->uncore, PCH_3DCGDIS1,
165 VFMUNIT_CLOCK_GATE_DISABLE);
166
167 /*
168 * According to the spec the following bits should be set in
169 * order to enable memory self-refresh
170 * The bit 22/21 of 0x42004
171 * The bit 5 of 0x42020
172 * The bit 15 of 0x45000
173 */
174 intel_uncore_write(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN2,
175 val: (intel_uncore_read(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN2) |
176 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
177 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
178 intel_uncore_write(uncore: &i915->uncore, DISP_ARB_CTL,
179 val: (intel_uncore_read(uncore: &i915->uncore, DISP_ARB_CTL) |
180 DISP_FBC_WM_DIS));
181
182 /*
183 * Based on the document from hardware guys the following bits
184 * should be set unconditionally in order to enable FBC.
185 * The bit 22 of 0x42000
186 * The bit 22 of 0x42004
187 * The bit 7,8,9 of 0x42020.
188 */
189 if (IS_IRONLAKE_M(i915)) {
190 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
191 intel_uncore_rmw(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN1, clear: 0, ILK_FBCQ_DIS);
192 intel_uncore_rmw(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN2, clear: 0, ILK_DPARB_GATE);
193 }
194
195 intel_uncore_write(uncore: &i915->uncore, ILK_DSPCLK_GATE_D, val: dspclk_gate);
196
197 intel_uncore_rmw(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN2, clear: 0, ILK_ELPIN_409_SELECT);
198
199 g4x_disable_trickle_feed(dev_priv: i915);
200
201 ibx_init_clock_gating(i915);
202}
203
204static void cpt_init_clock_gating(struct drm_i915_private *i915)
205{
206 struct intel_display *display = i915->display;
207 enum pipe pipe;
208 u32 val;
209
210 /*
211 * On Ibex Peak and Cougar Point, we need to disable clock
212 * gating for the panel power sequencer or it will fail to
213 * start up when no ports are active.
214 */
215 intel_uncore_write(uncore: &i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
216 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
217 PCH_CPUNIT_CLOCK_GATE_DISABLE);
218 intel_uncore_rmw(uncore: &i915->uncore, SOUTH_CHICKEN2, clear: 0, DPLS_EDP_PPS_FIX_DIS);
219 /* The below fixes the weird display corruption, a few pixels shifted
220 * downward, on (only) LVDS of some HP laptops with IVY.
221 */
222 for_each_pipe(display, pipe) {
223 val = intel_uncore_read(uncore: &i915->uncore, TRANS_CHICKEN2(pipe));
224 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
225 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
226 if (display->vbt.fdi_rx_polarity_inverted)
227 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
228 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
229 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
230 intel_uncore_write(uncore: &i915->uncore, TRANS_CHICKEN2(pipe), val);
231 }
232 /* WADP0ClockGatingDisable */
233 for_each_pipe(display, pipe) {
234 intel_uncore_write(uncore: &i915->uncore, TRANS_CHICKEN1(pipe),
235 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
236 }
237}
238
239static void gen6_check_mch_setup(struct drm_i915_private *i915)
240{
241 u32 tmp;
242
243 tmp = intel_uncore_read(uncore: &i915->uncore, MCH_SSKPD);
244 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
245 drm_dbg_kms(&i915->drm,
246 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
247 tmp);
248}
249
250static void gen6_init_clock_gating(struct drm_i915_private *i915)
251{
252 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
253
254 intel_uncore_write(uncore: &i915->uncore, ILK_DSPCLK_GATE_D, val: dspclk_gate);
255
256 intel_uncore_rmw(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN2, clear: 0, ILK_ELPIN_409_SELECT);
257
258 intel_uncore_write(uncore: &i915->uncore, GEN6_UCGCTL1,
259 val: intel_uncore_read(uncore: &i915->uncore, GEN6_UCGCTL1) |
260 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
261 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
262
263 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
264 * gating disable must be set. Failure to set it results in
265 * flickering pixels due to Z write ordering failures after
266 * some amount of runtime in the Mesa "fire" demo, and Unigine
267 * Sanctuary and Tropics, and apparently anything else with
268 * alpha test or pixel discard.
269 *
270 * According to the spec, bit 11 (RCCUNIT) must also be set,
271 * but we didn't debug actual testcases to find it out.
272 *
273 * WaDisableRCCUnitClockGating:snb
274 * WaDisableRCPBUnitClockGating:snb
275 */
276 intel_uncore_write(uncore: &i915->uncore, GEN6_UCGCTL2,
277 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
278 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
279
280 /*
281 * According to the spec the following bits should be
282 * set in order to enable memory self-refresh and fbc:
283 * The bit21 and bit22 of 0x42000
284 * The bit21 and bit22 of 0x42004
285 * The bit5 and bit7 of 0x42020
286 * The bit14 of 0x70180
287 * The bit14 of 0x71180
288 *
289 * WaFbcAsynchFlipDisableFbcQueue:snb
290 */
291 intel_uncore_write(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN1,
292 val: intel_uncore_read(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN1) |
293 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
294 intel_uncore_write(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN2,
295 val: intel_uncore_read(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN2) |
296 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
297 intel_uncore_write(uncore: &i915->uncore, ILK_DSPCLK_GATE_D,
298 val: intel_uncore_read(uncore: &i915->uncore, ILK_DSPCLK_GATE_D) |
299 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
300 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
301
302 g4x_disable_trickle_feed(dev_priv: i915);
303
304 cpt_init_clock_gating(i915);
305
306 gen6_check_mch_setup(i915);
307}
308
309static void lpt_init_clock_gating(struct drm_i915_private *i915)
310{
311 struct intel_display *display = i915->display;
312
313 /*
314 * TODO: this bit should only be enabled when really needed, then
315 * disabled when not needed anymore in order to save power.
316 */
317 if (HAS_PCH_LPT_LP(display))
318 intel_uncore_rmw(uncore: &i915->uncore, SOUTH_DSPCLK_GATE_D,
319 clear: 0, PCH_LP_PARTITION_LEVEL_DISABLE);
320
321 /* WADPOClockGatingDisable:hsw */
322 intel_uncore_rmw(uncore: &i915->uncore, TRANS_CHICKEN1(PIPE_A),
323 clear: 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
324}
325
326static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
327 int general_prio_credits,
328 int high_prio_credits)
329{
330 u32 misccpctl;
331 u32 val;
332
333 /* WaTempDisableDOPClkGating:bdw */
334 misccpctl = intel_uncore_rmw(uncore: &i915->uncore, GEN7_MISCCPCTL,
335 GEN7_DOP_CLOCK_GATE_ENABLE, set: 0);
336
337 val = intel_gt_mcr_read_any(gt: to_gt(i915), GEN8_L3SQCREG1);
338 val &= ~L3_PRIO_CREDITS_MASK;
339 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
340 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
341 intel_gt_mcr_multicast_write(gt: to_gt(i915), GEN8_L3SQCREG1, value: val);
342
343 /*
344 * Wait at least 100 clocks before re-enabling clock gating.
345 * See the definition of L3SQCREG1 in BSpec.
346 */
347 intel_gt_mcr_read_any(gt: to_gt(i915), GEN8_L3SQCREG1);
348 udelay(usec: 1);
349 intel_uncore_write(uncore: &i915->uncore, GEN7_MISCCPCTL, val: misccpctl);
350}
351
352static void dg2_init_clock_gating(struct drm_i915_private *i915)
353{
354 /* Wa_22010954014:dg2 */
355 intel_uncore_rmw(uncore: &i915->uncore, XEHP_CLOCK_GATE_DIS, clear: 0,
356 SGSI_SIDECLK_DIS);
357}
358
359static void cnp_init_clock_gating(struct drm_i915_private *i915)
360{
361 struct intel_display *display = i915->display;
362
363 if (!HAS_PCH_CNP(display))
364 return;
365
366 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
367 intel_uncore_rmw(uncore: &i915->uncore, SOUTH_DSPCLK_GATE_D, clear: 0, CNP_PWM_CGE_GATING_DISABLE);
368}
369
370static void cfl_init_clock_gating(struct drm_i915_private *i915)
371{
372 cnp_init_clock_gating(i915);
373 gen9_init_clock_gating(i915);
374
375 /* WAC6entrylatency:cfl */
376 intel_uncore_rmw(uncore: &i915->uncore, FBC_LLC_READ_CTRL, clear: 0, FBC_LLC_FULLY_OPEN);
377
378 /*
379 * WaFbcTurnOffFbcWatermark:cfl
380 * Display WA #0562: cfl
381 */
382 intel_uncore_rmw(uncore: &i915->uncore, DISP_ARB_CTL, clear: 0, DISP_FBC_WM_DIS);
383}
384
385static void kbl_init_clock_gating(struct drm_i915_private *i915)
386{
387 gen9_init_clock_gating(i915);
388
389 /* WAC6entrylatency:kbl */
390 intel_uncore_rmw(uncore: &i915->uncore, FBC_LLC_READ_CTRL, clear: 0, FBC_LLC_FULLY_OPEN);
391
392 /* WaDisableSDEUnitClockGating:kbl */
393 if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
394 intel_uncore_rmw(uncore: &i915->uncore, GEN8_UCGCTL6,
395 clear: 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
396
397 /* WaDisableGamClockGating:kbl */
398 if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
399 intel_uncore_rmw(uncore: &i915->uncore, GEN6_UCGCTL1,
400 clear: 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
401
402 /*
403 * WaFbcTurnOffFbcWatermark:kbl
404 * Display WA #0562: kbl
405 */
406 intel_uncore_rmw(uncore: &i915->uncore, DISP_ARB_CTL, clear: 0, DISP_FBC_WM_DIS);
407}
408
409static void skl_init_clock_gating(struct drm_i915_private *i915)
410{
411 gen9_init_clock_gating(i915);
412
413 /* WaDisableDopClockGating:skl */
414 intel_uncore_rmw(uncore: &i915->uncore, GEN7_MISCCPCTL,
415 GEN7_DOP_CLOCK_GATE_ENABLE, set: 0);
416
417 /* WAC6entrylatency:skl */
418 intel_uncore_rmw(uncore: &i915->uncore, FBC_LLC_READ_CTRL, clear: 0, FBC_LLC_FULLY_OPEN);
419
420 /*
421 * WaFbcTurnOffFbcWatermark:skl
422 * Display WA #0562: skl
423 */
424 intel_uncore_rmw(uncore: &i915->uncore, DISP_ARB_CTL, clear: 0, DISP_FBC_WM_DIS);
425}
426
427static void bdw_init_clock_gating(struct drm_i915_private *i915)
428{
429 struct intel_display *display = i915->display;
430 enum pipe pipe;
431
432 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
433 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PIPESL_1(PIPE_A), clear: 0, HSW_FBCQ_DIS);
434
435 /* WaSwitchSolVfFArbitrationPriority:bdw */
436 intel_uncore_rmw(uncore: &i915->uncore, GAM_ECOCHK, clear: 0, HSW_ECOCHK_ARB_PRIO_SOL);
437
438 /* WaPsrDPAMaskVBlankInSRD:bdw */
439 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PAR1_1, clear: 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
440
441 for_each_pipe(display, pipe) {
442 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
443 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PIPESL_1(pipe),
444 clear: 0, BDW_UNMASK_VBL_TO_REGS_IN_SRD);
445 }
446
447 /* WaVSRefCountFullforceMissDisable:bdw */
448 /* WaDSRefCountFullforceMissDisable:bdw */
449 intel_uncore_rmw(uncore: &i915->uncore, GEN7_FF_THREAD_MODE,
450 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, set: 0);
451
452 intel_uncore_write(uncore: &i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
453 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
454
455 /* WaDisableSDEUnitClockGating:bdw */
456 intel_uncore_rmw(uncore: &i915->uncore, GEN8_UCGCTL6, clear: 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
457
458 /* WaProgramL3SqcReg1Default:bdw */
459 gen8_set_l3sqc_credits(i915, general_prio_credits: 30, high_prio_credits: 2);
460
461 /* WaKVMNotificationOnConfigChange:bdw */
462 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PAR2_1,
463 clear: 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
464
465 lpt_init_clock_gating(i915);
466
467 /* WaDisableDopClockGating:bdw
468 *
469 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
470 * clock gating.
471 */
472 intel_uncore_rmw(uncore: &i915->uncore, GEN6_UCGCTL1, clear: 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
473}
474
475static void hsw_init_clock_gating(struct drm_i915_private *i915)
476{
477 struct intel_display *display = i915->display;
478 enum pipe pipe;
479
480 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
481 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PIPESL_1(PIPE_A), clear: 0, HSW_FBCQ_DIS);
482
483 /* WaPsrDPAMaskVBlankInSRD:hsw */
484 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PAR1_1, clear: 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
485
486 for_each_pipe(display, pipe) {
487 /* WaPsrDPRSUnmaskVBlankInSRD:hsw */
488 intel_uncore_rmw(uncore: &i915->uncore, CHICKEN_PIPESL_1(pipe),
489 clear: 0, HSW_UNMASK_VBL_TO_REGS_IN_SRD);
490 }
491
492 /* This is required by WaCatErrorRejectionIssue:hsw */
493 intel_uncore_rmw(uncore: &i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
494 clear: 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
495
496 /* WaSwitchSolVfFArbitrationPriority:hsw */
497 intel_uncore_rmw(uncore: &i915->uncore, GAM_ECOCHK, clear: 0, HSW_ECOCHK_ARB_PRIO_SOL);
498
499 lpt_init_clock_gating(i915);
500}
501
502static void ivb_init_clock_gating(struct drm_i915_private *i915)
503{
504 struct intel_display *display = i915->display;
505
506 intel_uncore_write(uncore: &i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
507
508 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
509 intel_uncore_rmw(uncore: &i915->uncore, ILK_DISPLAY_CHICKEN1, clear: 0, ILK_FBCQ_DIS);
510
511 /* WaDisableBackToBackFlipFix:ivb */
512 intel_uncore_write(uncore: &i915->uncore, IVB_CHICKEN3,
513 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
514 CHICKEN3_DGMG_DONE_FIX_DISABLE);
515
516 if (INTEL_INFO(i915)->gt == 1)
517 intel_uncore_write(uncore: &i915->uncore, GEN7_ROW_CHICKEN2,
518 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
519 else {
520 /* must write both registers */
521 intel_uncore_write(uncore: &i915->uncore, GEN7_ROW_CHICKEN2,
522 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
523 intel_uncore_write(uncore: &i915->uncore, GEN7_ROW_CHICKEN2_GT2,
524 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
525 }
526
527 /*
528 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
529 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
530 */
531 intel_uncore_write(uncore: &i915->uncore, GEN6_UCGCTL2,
532 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
533
534 /* This is required by WaCatErrorRejectionIssue:ivb */
535 intel_uncore_rmw(uncore: &i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
536 clear: 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
537
538 g4x_disable_trickle_feed(dev_priv: i915);
539
540 intel_uncore_rmw(uncore: &i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
541 GEN6_MBC_SNPCR_MED);
542
543 if (!HAS_PCH_NOP(display))
544 cpt_init_clock_gating(i915);
545
546 gen6_check_mch_setup(i915);
547}
548
549static void vlv_init_clock_gating(struct drm_i915_private *i915)
550{
551 /* WaDisableBackToBackFlipFix:vlv */
552 intel_uncore_write(uncore: &i915->uncore, IVB_CHICKEN3,
553 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
554 CHICKEN3_DGMG_DONE_FIX_DISABLE);
555
556 /* WaDisableDopClockGating:vlv */
557 intel_uncore_write(uncore: &i915->uncore, GEN7_ROW_CHICKEN2,
558 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
559
560 /* This is required by WaCatErrorRejectionIssue:vlv */
561 intel_uncore_rmw(uncore: &i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
562 clear: 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
563
564 /*
565 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
566 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
567 */
568 intel_uncore_write(uncore: &i915->uncore, GEN6_UCGCTL2,
569 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
570
571 /* WaDisableL3Bank2xClockGate:vlv
572 * Disabling L3 clock gating- MMIO 940c[25] = 1
573 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
574 intel_uncore_rmw(uncore: &i915->uncore, GEN7_UCGCTL4, clear: 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
575
576 /*
577 * WaDisableVLVClockGating_VBIIssue:vlv
578 * Disable clock gating on th GCFG unit to prevent a delay
579 * in the reporting of vblank events.
580 */
581 intel_uncore_write(uncore: &i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
582}
583
584static void chv_init_clock_gating(struct drm_i915_private *i915)
585{
586 /* WaVSRefCountFullforceMissDisable:chv */
587 /* WaDSRefCountFullforceMissDisable:chv */
588 intel_uncore_rmw(uncore: &i915->uncore, GEN7_FF_THREAD_MODE,
589 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, set: 0);
590
591 /* WaDisableSemaphoreAndSyncFlipWait:chv */
592 intel_uncore_write(uncore: &i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
593 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
594
595 /* WaDisableCSUnitClockGating:chv */
596 intel_uncore_rmw(uncore: &i915->uncore, GEN6_UCGCTL1, clear: 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
597
598 /* WaDisableSDEUnitClockGating:chv */
599 intel_uncore_rmw(uncore: &i915->uncore, GEN8_UCGCTL6, clear: 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
600
601 /*
602 * WaProgramL3SqcReg1Default:chv
603 * See gfxspecs/Related Documents/Performance Guide/
604 * LSQC Setting Recommendations.
605 */
606 gen8_set_l3sqc_credits(i915, general_prio_credits: 38, high_prio_credits: 2);
607}
608
609static void g4x_init_clock_gating(struct drm_i915_private *i915)
610{
611 u32 dspclk_gate;
612
613 intel_uncore_write(uncore: &i915->uncore, RENCLK_GATE_D1, val: 0);
614 intel_uncore_write(uncore: &i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
615 GS_UNIT_CLOCK_GATE_DISABLE |
616 CL_UNIT_CLOCK_GATE_DISABLE);
617 intel_uncore_write(uncore: &i915->uncore, RAMCLK_GATE_D, val: 0);
618 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
619 OVRUNIT_CLOCK_GATE_DISABLE |
620 OVCUNIT_CLOCK_GATE_DISABLE;
621 if (IS_GM45(i915))
622 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
623 intel_uncore_write(uncore: &i915->uncore, DSPCLK_GATE_D, val: dspclk_gate);
624
625 g4x_disable_trickle_feed(dev_priv: i915);
626}
627
628static void i965gm_init_clock_gating(struct drm_i915_private *i915)
629{
630 struct intel_uncore *uncore = &i915->uncore;
631
632 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
633 intel_uncore_write(uncore, RENCLK_GATE_D2, val: 0);
634 intel_uncore_write(uncore, DSPCLK_GATE_D, val: 0);
635 intel_uncore_write(uncore, RAMCLK_GATE_D, val: 0);
636 intel_uncore_write16(uncore, DEUC, val: 0);
637 intel_uncore_write(uncore,
638 MI_ARB_STATE,
639 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
640}
641
642static void i965g_init_clock_gating(struct drm_i915_private *i915)
643{
644 intel_uncore_write(uncore: &i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
645 I965_RCC_CLOCK_GATE_DISABLE |
646 I965_RCPB_CLOCK_GATE_DISABLE |
647 I965_ISC_CLOCK_GATE_DISABLE |
648 I965_FBC_CLOCK_GATE_DISABLE);
649 intel_uncore_write(uncore: &i915->uncore, RENCLK_GATE_D2, val: 0);
650 intel_uncore_write(uncore: &i915->uncore, MI_ARB_STATE,
651 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
652}
653
654static void gen3_init_clock_gating(struct drm_i915_private *i915)
655{
656 u32 dstate = intel_uncore_read(uncore: &i915->uncore, D_STATE);
657
658 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
659 DSTATE_DOT_CLOCK_GATING;
660 intel_uncore_write(uncore: &i915->uncore, D_STATE, val: dstate);
661
662 if (IS_PINEVIEW(i915))
663 intel_uncore_write(uncore: &i915->uncore, ECOSKPD(RENDER_RING_BASE),
664 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
665
666 /* IIR "flip pending" means done if this bit is set */
667 intel_uncore_write(uncore: &i915->uncore, ECOSKPD(RENDER_RING_BASE),
668 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
669
670 /* interrupts should cause a wake up from C3 */
671 intel_uncore_write(uncore: &i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
672
673 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
674 intel_uncore_write(uncore: &i915->uncore, MI_ARB_STATE,
675 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
676
677 intel_uncore_write(uncore: &i915->uncore, MI_ARB_STATE,
678 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
679}
680
681static void i85x_init_clock_gating(struct drm_i915_private *i915)
682{
683 intel_uncore_write(uncore: &i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
684
685 /* interrupts should cause a wake up from C3 */
686 intel_uncore_write(uncore: &i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
687 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
688
689 intel_uncore_write(uncore: &i915->uncore, MEM_MODE,
690 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
691
692 /*
693 * Have FBC ignore 3D activity since we use software
694 * render tracking, and otherwise a pure 3D workload
695 * (even if it just renders a single frame and then does
696 * absolutely nothing) would not allow FBC to recompress
697 * until a 2D blit occurs.
698 */
699 intel_uncore_write(uncore: &i915->uncore, SCPD0,
700 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
701}
702
703static void i830_init_clock_gating(struct drm_i915_private *i915)
704{
705 intel_uncore_write(uncore: &i915->uncore, MEM_MODE,
706 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
707 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
708}
709
710void intel_clock_gating_init(struct drm_i915_private *i915)
711{
712 i915->clock_gating_funcs->init_clock_gating(i915);
713}
714
715static void nop_init_clock_gating(struct drm_i915_private *i915)
716{
717 drm_dbg_kms(&i915->drm,
718 "No clock gating settings or workarounds applied.\n");
719}
720
721#define CG_FUNCS(platform) \
722static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
723 .init_clock_gating = platform##_init_clock_gating, \
724}
725
726CG_FUNCS(dg2);
727CG_FUNCS(cfl);
728CG_FUNCS(skl);
729CG_FUNCS(kbl);
730CG_FUNCS(bxt);
731CG_FUNCS(glk);
732CG_FUNCS(bdw);
733CG_FUNCS(chv);
734CG_FUNCS(hsw);
735CG_FUNCS(ivb);
736CG_FUNCS(vlv);
737CG_FUNCS(gen6);
738CG_FUNCS(ilk);
739CG_FUNCS(g4x);
740CG_FUNCS(i965gm);
741CG_FUNCS(i965g);
742CG_FUNCS(gen3);
743CG_FUNCS(i85x);
744CG_FUNCS(i830);
745CG_FUNCS(nop);
746#undef CG_FUNCS
747
748/**
749 * intel_clock_gating_hooks_init - setup the clock gating hooks
750 * @i915: device private
751 *
752 * Setup the hooks that configure which clocks of a given platform can be
753 * gated and also apply various GT and display specific workarounds for these
754 * platforms. Note that some GT specific workarounds are applied separately
755 * when GPU contexts or batchbuffers start their execution.
756 */
757void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
758{
759 if (IS_DG2(i915))
760 i915->clock_gating_funcs = &dg2_clock_gating_funcs;
761 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
762 i915->clock_gating_funcs = &cfl_clock_gating_funcs;
763 else if (IS_SKYLAKE(i915))
764 i915->clock_gating_funcs = &skl_clock_gating_funcs;
765 else if (IS_KABYLAKE(i915))
766 i915->clock_gating_funcs = &kbl_clock_gating_funcs;
767 else if (IS_BROXTON(i915))
768 i915->clock_gating_funcs = &bxt_clock_gating_funcs;
769 else if (IS_GEMINILAKE(i915))
770 i915->clock_gating_funcs = &glk_clock_gating_funcs;
771 else if (IS_BROADWELL(i915))
772 i915->clock_gating_funcs = &bdw_clock_gating_funcs;
773 else if (IS_CHERRYVIEW(i915))
774 i915->clock_gating_funcs = &chv_clock_gating_funcs;
775 else if (IS_HASWELL(i915))
776 i915->clock_gating_funcs = &hsw_clock_gating_funcs;
777 else if (IS_IVYBRIDGE(i915))
778 i915->clock_gating_funcs = &ivb_clock_gating_funcs;
779 else if (IS_VALLEYVIEW(i915))
780 i915->clock_gating_funcs = &vlv_clock_gating_funcs;
781 else if (GRAPHICS_VER(i915) == 6)
782 i915->clock_gating_funcs = &gen6_clock_gating_funcs;
783 else if (GRAPHICS_VER(i915) == 5)
784 i915->clock_gating_funcs = &ilk_clock_gating_funcs;
785 else if (IS_G4X(i915))
786 i915->clock_gating_funcs = &g4x_clock_gating_funcs;
787 else if (IS_I965GM(i915))
788 i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
789 else if (IS_I965G(i915))
790 i915->clock_gating_funcs = &i965g_clock_gating_funcs;
791 else if (GRAPHICS_VER(i915) == 3)
792 i915->clock_gating_funcs = &gen3_clock_gating_funcs;
793 else if (IS_I85X(i915) || IS_I865G(i915))
794 i915->clock_gating_funcs = &i85x_clock_gating_funcs;
795 else if (GRAPHICS_VER(i915) == 2)
796 i915->clock_gating_funcs = &i830_clock_gating_funcs;
797 else
798 i915->clock_gating_funcs = &nop_clock_gating_funcs;
799}
800