1/*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/debugfs.h>
26#include <linux/firmware.h>
27#include <drm/drm_vblank.h>
28
29#include <drm/drm_file.h>
30#include <drm/drm_print.h>
31
32#include "i915_reg.h"
33#include "i915_utils.h"
34#include "intel_crtc.h"
35#include "intel_de.h"
36#include "intel_display_power_well.h"
37#include "intel_display_regs.h"
38#include "intel_display_rpm.h"
39#include "intel_display_types.h"
40#include "intel_dmc.h"
41#include "intel_dmc_regs.h"
42#include "intel_flipq.h"
43#include "intel_step.h"
44
45/**
46 * DOC: DMC Firmware Support
47 *
48 * From gen9 onwards we have newly added DMC (Display microcontroller) in display
49 * engine to save and restore the state of display engine when it enter into
50 * low-power state and comes back to normal.
51 */
52
53#define INTEL_DMC_FIRMWARE_URL "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"
54
55enum intel_dmc_id {
56 DMC_FW_MAIN = 0,
57 DMC_FW_PIPEA,
58 DMC_FW_PIPEB,
59 DMC_FW_PIPEC,
60 DMC_FW_PIPED,
61 DMC_FW_MAX
62};
63
64struct intel_dmc {
65 struct intel_display *display;
66 struct work_struct work;
67 const char *fw_path;
68 u32 max_fw_size; /* bytes */
69 u32 version;
70 struct {
71 u32 dc5_start;
72 u32 count;
73 } dc6_allowed;
74 struct dmc_fw_info {
75 u32 mmio_count;
76 i915_reg_t mmioaddr[20];
77 u32 mmiodata[20];
78 u32 dmc_offset;
79 u32 start_mmioaddr;
80 u32 dmc_fw_size; /*dwords */
81 u32 *payload;
82 bool present;
83 } dmc_info[DMC_FW_MAX];
84};
85
86/* Note: This may be NULL. */
87static struct intel_dmc *display_to_dmc(struct intel_display *display)
88{
89 return display->dmc.dmc;
90}
91
92static const char *dmc_firmware_param(struct intel_display *display)
93{
94 const char *p = display->params.dmc_firmware_path;
95
96 return p && *p ? p : NULL;
97}
98
99static bool dmc_firmware_param_disabled(struct intel_display *display)
100{
101 const char *p = dmc_firmware_param(display);
102
103 /* Magic path to indicate disabled */
104 return p && !strcmp(p, "/dev/null");
105}
106
107#define DMC_VERSION(major, minor) ((major) << 16 | (minor))
108#define DMC_VERSION_MAJOR(version) ((version) >> 16)
109#define DMC_VERSION_MINOR(version) ((version) & 0xffff)
110
111#define DMC_PATH(platform) \
112 "i915/" __stringify(platform) "_dmc.bin"
113
114/*
115 * New DMC additions should not use this. This is used solely to remain
116 * compatible with systems that have not yet updated DMC blobs to use
117 * unversioned file names.
118 */
119#define DMC_LEGACY_PATH(platform, major, minor) \
120 "i915/" \
121 __stringify(platform) "_dmc_ver" \
122 __stringify(major) "_" \
123 __stringify(minor) ".bin"
124
125#define XE2LPD_DMC_MAX_FW_SIZE 0x8000
126#define XELPDP_DMC_MAX_FW_SIZE 0x7000
127#define DISPLAY_VER13_DMC_MAX_FW_SIZE 0x20000
128#define DISPLAY_VER12_DMC_MAX_FW_SIZE ICL_DMC_MAX_FW_SIZE
129
130#define XE3LPD_DMC_PATH DMC_PATH(xe3lpd)
131MODULE_FIRMWARE(XE3LPD_DMC_PATH);
132
133#define XE2LPD_DMC_PATH DMC_PATH(xe2lpd)
134MODULE_FIRMWARE(XE2LPD_DMC_PATH);
135
136#define BMG_DMC_PATH DMC_PATH(bmg)
137MODULE_FIRMWARE(BMG_DMC_PATH);
138
139#define MTL_DMC_PATH DMC_PATH(mtl)
140MODULE_FIRMWARE(MTL_DMC_PATH);
141
142#define DG2_DMC_PATH DMC_LEGACY_PATH(dg2, 2, 08)
143MODULE_FIRMWARE(DG2_DMC_PATH);
144
145#define ADLP_DMC_PATH DMC_PATH(adlp)
146#define ADLP_DMC_FALLBACK_PATH DMC_LEGACY_PATH(adlp, 2, 16)
147MODULE_FIRMWARE(ADLP_DMC_PATH);
148MODULE_FIRMWARE(ADLP_DMC_FALLBACK_PATH);
149
150#define ADLS_DMC_PATH DMC_LEGACY_PATH(adls, 2, 01)
151MODULE_FIRMWARE(ADLS_DMC_PATH);
152
153#define DG1_DMC_PATH DMC_LEGACY_PATH(dg1, 2, 02)
154MODULE_FIRMWARE(DG1_DMC_PATH);
155
156#define RKL_DMC_PATH DMC_LEGACY_PATH(rkl, 2, 03)
157MODULE_FIRMWARE(RKL_DMC_PATH);
158
159#define TGL_DMC_PATH DMC_LEGACY_PATH(tgl, 2, 12)
160MODULE_FIRMWARE(TGL_DMC_PATH);
161
162#define ICL_DMC_PATH DMC_LEGACY_PATH(icl, 1, 09)
163#define ICL_DMC_MAX_FW_SIZE 0x6000
164MODULE_FIRMWARE(ICL_DMC_PATH);
165
166#define GLK_DMC_PATH DMC_LEGACY_PATH(glk, 1, 04)
167#define GLK_DMC_MAX_FW_SIZE 0x4000
168MODULE_FIRMWARE(GLK_DMC_PATH);
169
170#define KBL_DMC_PATH DMC_LEGACY_PATH(kbl, 1, 04)
171#define KBL_DMC_MAX_FW_SIZE BXT_DMC_MAX_FW_SIZE
172MODULE_FIRMWARE(KBL_DMC_PATH);
173
174#define SKL_DMC_PATH DMC_LEGACY_PATH(skl, 1, 27)
175#define SKL_DMC_MAX_FW_SIZE BXT_DMC_MAX_FW_SIZE
176MODULE_FIRMWARE(SKL_DMC_PATH);
177
178#define BXT_DMC_PATH DMC_LEGACY_PATH(bxt, 1, 07)
179#define BXT_DMC_MAX_FW_SIZE 0x3000
180MODULE_FIRMWARE(BXT_DMC_PATH);
181
182static const char *dmc_firmware_default(struct intel_display *display, u32 *size)
183{
184 const char *fw_path = NULL;
185 u32 max_fw_size = 0;
186
187 if (DISPLAY_VERx100(display) == 3002 ||
188 DISPLAY_VERx100(display) == 3000) {
189 fw_path = XE3LPD_DMC_PATH;
190 max_fw_size = XE2LPD_DMC_MAX_FW_SIZE;
191 } else if (DISPLAY_VERx100(display) == 2000) {
192 fw_path = XE2LPD_DMC_PATH;
193 max_fw_size = XE2LPD_DMC_MAX_FW_SIZE;
194 } else if (DISPLAY_VERx100(display) == 1401) {
195 fw_path = BMG_DMC_PATH;
196 max_fw_size = XELPDP_DMC_MAX_FW_SIZE;
197 } else if (DISPLAY_VERx100(display) == 1400) {
198 fw_path = MTL_DMC_PATH;
199 max_fw_size = XELPDP_DMC_MAX_FW_SIZE;
200 } else if (display->platform.dg2) {
201 fw_path = DG2_DMC_PATH;
202 max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
203 } else if (display->platform.alderlake_p) {
204 fw_path = ADLP_DMC_PATH;
205 max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
206 } else if (display->platform.alderlake_s) {
207 fw_path = ADLS_DMC_PATH;
208 max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
209 } else if (display->platform.dg1) {
210 fw_path = DG1_DMC_PATH;
211 max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
212 } else if (display->platform.rocketlake) {
213 fw_path = RKL_DMC_PATH;
214 max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
215 } else if (display->platform.tigerlake) {
216 fw_path = TGL_DMC_PATH;
217 max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
218 } else if (DISPLAY_VER(display) == 11) {
219 fw_path = ICL_DMC_PATH;
220 max_fw_size = ICL_DMC_MAX_FW_SIZE;
221 } else if (display->platform.geminilake) {
222 fw_path = GLK_DMC_PATH;
223 max_fw_size = GLK_DMC_MAX_FW_SIZE;
224 } else if (display->platform.kabylake ||
225 display->platform.coffeelake ||
226 display->platform.cometlake) {
227 fw_path = KBL_DMC_PATH;
228 max_fw_size = KBL_DMC_MAX_FW_SIZE;
229 } else if (display->platform.skylake) {
230 fw_path = SKL_DMC_PATH;
231 max_fw_size = SKL_DMC_MAX_FW_SIZE;
232 } else if (display->platform.broxton) {
233 fw_path = BXT_DMC_PATH;
234 max_fw_size = BXT_DMC_MAX_FW_SIZE;
235 }
236
237 *size = max_fw_size;
238
239 return fw_path;
240}
241
242#define DMC_DEFAULT_FW_OFFSET 0xFFFFFFFF
243#define PACKAGE_MAX_FW_INFO_ENTRIES 20
244#define PACKAGE_V2_MAX_FW_INFO_ENTRIES 32
245#define DMC_V1_MAX_MMIO_COUNT 8
246#define DMC_V3_MAX_MMIO_COUNT 20
247#define DMC_V1_MMIO_START_RANGE 0x80000
248
249#define PIPE_TO_DMC_ID(pipe) (DMC_FW_PIPEA + ((pipe) - PIPE_A))
250
251struct intel_css_header {
252 /* 0x09 for DMC */
253 u32 module_type;
254
255 /* Includes the DMC specific header in dwords */
256 u32 header_len;
257
258 /* always value would be 0x10000 */
259 u32 header_ver;
260
261 /* Not used */
262 u32 module_id;
263
264 /* Not used */
265 u32 module_vendor;
266
267 /* in YYYYMMDD format */
268 u32 date;
269
270 /* Size in dwords (CSS_Headerlen + PackageHeaderLen + dmc FWsLen)/4 */
271 u32 size;
272
273 /* Not used */
274 u32 key_size;
275
276 /* Not used */
277 u32 modulus_size;
278
279 /* Not used */
280 u32 exponent_size;
281
282 /* Not used */
283 u32 reserved1[12];
284
285 /* Major Minor */
286 u32 version;
287
288 /* Not used */
289 u32 reserved2[8];
290
291 /* Not used */
292 u32 kernel_header_info;
293} __packed;
294
295struct intel_fw_info {
296 u8 reserved1;
297
298 /* reserved on package_header version 1, must be 0 on version 2 */
299 u8 dmc_id;
300
301 /* Stepping (A, B, C, ..., *). * is a wildcard */
302 char stepping;
303
304 /* Sub-stepping (0, 1, ..., *). * is a wildcard */
305 char substepping;
306
307 u32 offset;
308 u32 reserved2;
309} __packed;
310
311struct intel_package_header {
312 /* DMC container header length in dwords */
313 u8 header_len;
314
315 /* 0x01, 0x02 */
316 u8 header_ver;
317
318 u8 reserved[10];
319
320 /* Number of valid entries in the FWInfo array below */
321 u32 num_entries;
322} __packed;
323
324struct intel_dmc_header_base {
325 /* always value would be 0x40403E3E */
326 u32 signature;
327
328 /* DMC binary header length */
329 u8 header_len;
330
331 /* 0x01 */
332 u8 header_ver;
333
334 /* Reserved */
335 u16 dmcc_ver;
336
337 /* Major, Minor */
338 u32 project;
339
340 /* Firmware program size (excluding header) in dwords */
341 u32 fw_size;
342
343 /* Major Minor version */
344 u32 fw_version;
345} __packed;
346
347struct intel_dmc_header_v1 {
348 struct intel_dmc_header_base base;
349
350 /* Number of valid MMIO cycles present. */
351 u32 mmio_count;
352
353 /* MMIO address */
354 u32 mmioaddr[DMC_V1_MAX_MMIO_COUNT];
355
356 /* MMIO data */
357 u32 mmiodata[DMC_V1_MAX_MMIO_COUNT];
358
359 /* FW filename */
360 char dfile[32];
361
362 u32 reserved1[2];
363} __packed;
364
365struct intel_dmc_header_v3 {
366 struct intel_dmc_header_base base;
367
368 /* DMC RAM start MMIO address */
369 u32 start_mmioaddr;
370
371 u32 reserved[9];
372
373 /* FW filename */
374 char dfile[32];
375
376 /* Number of valid MMIO cycles present. */
377 u32 mmio_count;
378
379 /* MMIO address */
380 u32 mmioaddr[DMC_V3_MAX_MMIO_COUNT];
381
382 /* MMIO data */
383 u32 mmiodata[DMC_V3_MAX_MMIO_COUNT];
384} __packed;
385
386struct stepping_info {
387 char stepping;
388 char substepping;
389};
390
391#define for_each_dmc_id(__dmc_id) \
392 for ((__dmc_id) = DMC_FW_MAIN; (__dmc_id) < DMC_FW_MAX; (__dmc_id)++)
393
394static bool is_valid_dmc_id(enum intel_dmc_id dmc_id)
395{
396 return dmc_id >= DMC_FW_MAIN && dmc_id < DMC_FW_MAX;
397}
398
399static bool has_dmc_id_fw(struct intel_display *display, enum intel_dmc_id dmc_id)
400{
401 struct intel_dmc *dmc = display_to_dmc(display);
402
403 return dmc && dmc->dmc_info[dmc_id].payload;
404}
405
406bool intel_dmc_has_payload(struct intel_display *display)
407{
408 return has_dmc_id_fw(display, dmc_id: DMC_FW_MAIN);
409}
410
411static const struct stepping_info *
412intel_get_stepping_info(struct intel_display *display,
413 struct stepping_info *si)
414{
415 const char *step_name = intel_step_name(INTEL_DISPLAY_STEP(display));
416
417 si->stepping = step_name[0];
418 si->substepping = step_name[1];
419 return si;
420}
421
422static void gen9_set_dc_state_debugmask(struct intel_display *display)
423{
424 /* The below bit doesn't need to be cleared ever afterwards */
425 intel_de_rmw(display, DC_STATE_DEBUG, clear: 0,
426 DC_STATE_DEBUG_MASK_CORES | DC_STATE_DEBUG_MASK_MEMORY_UP);
427 intel_de_posting_read(display, DC_STATE_DEBUG);
428}
429
430static void disable_event_handler(struct intel_display *display,
431 i915_reg_t ctl_reg, i915_reg_t htp_reg)
432{
433 intel_de_write(display, reg: ctl_reg,
434 REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK,
435 DMC_EVT_CTL_TYPE_EDGE_0_1) |
436 REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK,
437 DMC_EVENT_FALSE));
438 intel_de_write(display, reg: htp_reg, val: 0);
439}
440
441static void disable_all_event_handlers(struct intel_display *display,
442 enum intel_dmc_id dmc_id)
443{
444 int handler;
445
446 /* TODO: disable the event handlers on pre-GEN12 platforms as well */
447 if (DISPLAY_VER(display) < 12)
448 return;
449
450 if (!has_dmc_id_fw(display, dmc_id))
451 return;
452
453 for (handler = 0; handler < DMC_EVENT_HANDLER_COUNT_GEN12; handler++)
454 disable_event_handler(display,
455 DMC_EVT_CTL(display, dmc_id, handler),
456 DMC_EVT_HTP(display, dmc_id, handler));
457}
458
459static void adlp_pipedmc_clock_gating_wa(struct intel_display *display, bool enable)
460{
461 enum pipe pipe;
462
463 /*
464 * Wa_16015201720:adl-p,dg2
465 * The WA requires clock gating to be disabled all the time
466 * for pipe A and B.
467 * For pipe C and D clock gating needs to be disabled only
468 * during initializing the firmware.
469 */
470 if (enable)
471 for (pipe = PIPE_A; pipe <= PIPE_D; pipe++)
472 intel_de_rmw(display, CLKGATE_DIS_PSL_EXT(pipe),
473 clear: 0, PIPEDMC_GATING_DIS);
474 else
475 for (pipe = PIPE_C; pipe <= PIPE_D; pipe++)
476 intel_de_rmw(display, CLKGATE_DIS_PSL_EXT(pipe),
477 PIPEDMC_GATING_DIS, set: 0);
478}
479
480static void mtl_pipedmc_clock_gating_wa(struct intel_display *display)
481{
482 /*
483 * Wa_16015201720
484 * The WA requires clock gating to be disabled all the time
485 * for pipe A and B.
486 */
487 intel_de_rmw(display, GEN9_CLKGATE_DIS_0, clear: 0,
488 MTL_PIPEDMC_GATING_DIS(PIPE_A) |
489 MTL_PIPEDMC_GATING_DIS(PIPE_B));
490}
491
492static void pipedmc_clock_gating_wa(struct intel_display *display, bool enable)
493{
494 if (display->platform.meteorlake && enable)
495 mtl_pipedmc_clock_gating_wa(display);
496 else if (DISPLAY_VER(display) == 13)
497 adlp_pipedmc_clock_gating_wa(display, enable);
498}
499
500static u32 pipedmc_interrupt_mask(struct intel_display *display)
501{
502 /*
503 * FIXME PIPEDMC_ERROR not enabled for now due to LNL pipe B
504 * triggering it during the first DC state transition. Figure
505 * out what is going on...
506 */
507 return PIPEDMC_FLIPQ_PROG_DONE |
508 PIPEDMC_GTT_FAULT |
509 PIPEDMC_ATS_FAULT;
510}
511
512static u32 dmc_evt_ctl_disable(void)
513{
514 return REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK,
515 DMC_EVT_CTL_TYPE_EDGE_0_1) |
516 REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK,
517 DMC_EVENT_FALSE);
518}
519
520static bool is_dmc_evt_ctl_reg(struct intel_display *display,
521 enum intel_dmc_id dmc_id, i915_reg_t reg)
522{
523 u32 offset = i915_mmio_reg_offset(reg);
524 u32 start = i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, 0));
525 u32 end = i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
526
527 return offset >= start && offset < end;
528}
529
530static bool is_dmc_evt_htp_reg(struct intel_display *display,
531 enum intel_dmc_id dmc_id, i915_reg_t reg)
532{
533 u32 offset = i915_mmio_reg_offset(reg);
534 u32 start = i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, 0));
535 u32 end = i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
536
537 return offset >= start && offset < end;
538}
539
540static bool is_event_handler(struct intel_display *display,
541 enum intel_dmc_id dmc_id,
542 unsigned int event_id,
543 i915_reg_t reg, u32 data)
544{
545 return is_dmc_evt_ctl_reg(display, dmc_id, reg) &&
546 REG_FIELD_GET(DMC_EVT_CTL_EVENT_ID_MASK, data) == event_id;
547}
548
549static bool disable_dmc_evt(struct intel_display *display,
550 enum intel_dmc_id dmc_id,
551 i915_reg_t reg, u32 data)
552{
553 if (!is_dmc_evt_ctl_reg(display, dmc_id, reg))
554 return false;
555
556 /* keep all pipe DMC events disabled by default */
557 if (dmc_id != DMC_FW_MAIN)
558 return true;
559
560 /* also disable the flip queue event on the main DMC on TGL */
561 if (display->platform.tigerlake &&
562 is_event_handler(display, dmc_id, event_id: MAINDMC_EVENT_CLK_MSEC, reg, data))
563 return true;
564
565 /* also disable the HRR event on the main DMC on TGL/ADLS */
566 if ((display->platform.tigerlake || display->platform.alderlake_s) &&
567 is_event_handler(display, dmc_id, event_id: MAINDMC_EVENT_VBLANK_A, reg, data))
568 return true;
569
570 return false;
571}
572
573static u32 dmc_mmiodata(struct intel_display *display,
574 struct intel_dmc *dmc,
575 enum intel_dmc_id dmc_id, int i)
576{
577 if (disable_dmc_evt(display, dmc_id,
578 reg: dmc->dmc_info[dmc_id].mmioaddr[i],
579 data: dmc->dmc_info[dmc_id].mmiodata[i]))
580 return dmc_evt_ctl_disable();
581 else
582 return dmc->dmc_info[dmc_id].mmiodata[i];
583}
584
585static void dmc_load_mmio(struct intel_display *display, enum intel_dmc_id dmc_id)
586{
587 struct intel_dmc *dmc = display_to_dmc(display);
588 int i;
589
590 for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
591 intel_de_write(display, reg: dmc->dmc_info[dmc_id].mmioaddr[i],
592 val: dmc_mmiodata(display, dmc, dmc_id, i));
593 }
594}
595
596static void dmc_load_program(struct intel_display *display, enum intel_dmc_id dmc_id)
597{
598 struct intel_dmc *dmc = display_to_dmc(display);
599 int i;
600
601 disable_all_event_handlers(display, dmc_id);
602
603 preempt_disable();
604
605 for (i = 0; i < dmc->dmc_info[dmc_id].dmc_fw_size; i++) {
606 intel_de_write_fw(display,
607 DMC_PROGRAM(dmc->dmc_info[dmc_id].start_mmioaddr, i),
608 val: dmc->dmc_info[dmc_id].payload[i]);
609 }
610
611 preempt_enable();
612
613 dmc_load_mmio(display, dmc_id);
614}
615
616static void assert_dmc_loaded(struct intel_display *display,
617 enum intel_dmc_id dmc_id)
618{
619 struct intel_dmc *dmc = display_to_dmc(display);
620 u32 expected, found;
621 int i;
622
623 if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(display, dmc_id))
624 return;
625
626 found = intel_de_read(display, DMC_PROGRAM(dmc->dmc_info[dmc_id].start_mmioaddr, 0));
627 expected = dmc->dmc_info[dmc_id].payload[0];
628
629 drm_WARN(display->drm, found != expected,
630 "DMC %d program storage start incorrect (expected 0x%x, current 0x%x)\n",
631 dmc_id, expected, found);
632
633 for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
634 i915_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
635
636 found = intel_de_read(display, reg);
637 expected = dmc_mmiodata(display, dmc, dmc_id, i);
638
639 /* once set DMC_EVT_CTL_ENABLE can't be cleared :/ */
640 if (is_dmc_evt_ctl_reg(display, dmc_id, reg)) {
641 found &= ~DMC_EVT_CTL_ENABLE;
642 expected &= ~DMC_EVT_CTL_ENABLE;
643 }
644
645 drm_WARN(display->drm, found != expected,
646 "DMC %d mmio[%d]/0x%x incorrect (expected 0x%x, current 0x%x)\n",
647 dmc_id, i, i915_mmio_reg_offset(reg), expected, found);
648 }
649}
650
651void assert_main_dmc_loaded(struct intel_display *display)
652{
653 assert_dmc_loaded(display, dmc_id: DMC_FW_MAIN);
654}
655
656static bool need_pipedmc_load_program(struct intel_display *display)
657{
658 /* On TGL/derivatives pipe DMC state is lost when PG1 is disabled */
659 return DISPLAY_VER(display) == 12;
660}
661
662static bool need_pipedmc_load_mmio(struct intel_display *display, enum pipe pipe)
663{
664 /*
665 * PTL:
666 * - pipe A/B DMC doesn't need save/restore
667 * - pipe C/D DMC is in PG0, needs manual save/restore
668 */
669 if (DISPLAY_VER(display) == 30)
670 return pipe >= PIPE_C;
671
672 /*
673 * FIXME LNL unclear, main DMC firmware has the pipe DMC A/B PG0
674 * save/restore, but so far unable to see the loss of pipe DMC state
675 * in action. Are we just failing to turn off PG0 due to some other
676 * SoC level stuff?
677 */
678 if (DISPLAY_VER(display) == 20)
679 return false;
680
681 /*
682 * FIXME BMG untested, main DMC firmware has the
683 * pipe DMC A/B PG0 save/restore...
684 */
685 if (display->platform.battlemage)
686 return false;
687
688 /*
689 * DG2:
690 * - Pipe DMCs presumably in PG0?
691 * - No DC6, and even DC9 doesn't seem to result
692 * in loss of DMC state for whatever reason
693 */
694 if (display->platform.dg2)
695 return false;
696
697 /*
698 * ADL/MTL:
699 * - pipe A/B DMC is in PG0, saved/restored by the main DMC
700 * - pipe C/D DMC is in PG0, needs manual save/restore
701 */
702 if (IS_DISPLAY_VER(display, 13, 14))
703 return pipe >= PIPE_C;
704
705 return false;
706}
707
708static bool can_enable_pipedmc(const struct intel_crtc_state *crtc_state)
709{
710 struct intel_display *display = to_intel_display(crtc_state);
711
712 /*
713 * On TGL/derivatives pipe DMC state is lost when PG1 is disabled.
714 * Do not even enable the pipe DMC when that can happen outside
715 * of driver control (PSR+DC5/6).
716 */
717 if (DISPLAY_VER(display) == 12 && crtc_state->has_psr)
718 return false;
719
720 return true;
721}
722
723void intel_dmc_enable_pipe(const struct intel_crtc_state *crtc_state)
724{
725 struct intel_display *display = to_intel_display(crtc_state);
726 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
727 enum pipe pipe = crtc->pipe;
728 enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
729
730 if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(display, dmc_id))
731 return;
732
733 if (!can_enable_pipedmc(crtc_state)) {
734 intel_dmc_disable_pipe(crtc_state);
735 return;
736 }
737
738 if (need_pipedmc_load_program(display))
739 dmc_load_program(display, dmc_id);
740 else if (need_pipedmc_load_mmio(display, pipe))
741 dmc_load_mmio(display, dmc_id);
742
743 assert_dmc_loaded(display, dmc_id);
744
745 if (DISPLAY_VER(display) >= 20) {
746 intel_flipq_reset(display, pipe);
747
748 intel_de_write(display, PIPEDMC_INTERRUPT(pipe), val: pipedmc_interrupt_mask(display));
749 intel_de_write(display, PIPEDMC_INTERRUPT_MASK(pipe), val: ~pipedmc_interrupt_mask(display));
750 }
751
752 if (DISPLAY_VER(display) >= 14)
753 intel_de_rmw(display, MTL_PIPEDMC_CONTROL, clear: 0, PIPEDMC_ENABLE_MTL(pipe));
754 else
755 intel_de_rmw(display, PIPEDMC_CONTROL(pipe), clear: 0, PIPEDMC_ENABLE);
756}
757
758void intel_dmc_disable_pipe(const struct intel_crtc_state *crtc_state)
759{
760 struct intel_display *display = to_intel_display(crtc_state);
761 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
762 enum pipe pipe = crtc->pipe;
763 enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
764
765 if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(display, dmc_id))
766 return;
767
768 if (DISPLAY_VER(display) >= 14)
769 intel_de_rmw(display, MTL_PIPEDMC_CONTROL, PIPEDMC_ENABLE_MTL(pipe), set: 0);
770 else
771 intel_de_rmw(display, PIPEDMC_CONTROL(pipe), PIPEDMC_ENABLE, set: 0);
772
773 if (DISPLAY_VER(display) >= 20) {
774 intel_de_write(display, PIPEDMC_INTERRUPT_MASK(pipe), val: ~0);
775 intel_de_write(display, PIPEDMC_INTERRUPT(pipe), val: pipedmc_interrupt_mask(display));
776
777 intel_flipq_reset(display, pipe);
778 }
779}
780
781static void dmc_configure_event(struct intel_display *display,
782 enum intel_dmc_id dmc_id,
783 unsigned int event_id,
784 bool enable)
785{
786 struct intel_dmc *dmc = display_to_dmc(display);
787 int num_handlers = 0;
788 int i;
789
790 for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
791 i915_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
792 u32 data = dmc->dmc_info[dmc_id].mmiodata[i];
793
794 if (!is_event_handler(display, dmc_id, event_id, reg, data))
795 continue;
796
797 intel_de_write(display, reg, val: enable ? data : dmc_evt_ctl_disable());
798 num_handlers++;
799 }
800
801 drm_WARN_ONCE(display->drm, num_handlers != 1,
802 "DMC %d has %d handlers for event 0x%x\n",
803 dmc_id, num_handlers, event_id);
804}
805
806/**
807 * intel_dmc_block_pkgc() - block PKG C-state
808 * @display: display instance
809 * @pipe: pipe which register use to block
810 * @block: block/unblock
811 *
812 * This interface is target for Wa_16025596647 usage. I.e. to set/clear
813 * PIPEDMC_BLOCK_PKGC_SW_BLOCK_PKGC_ALWAYS bit in PIPEDMC_BLOCK_PKGC_SW register.
814 */
815void intel_dmc_block_pkgc(struct intel_display *display, enum pipe pipe,
816 bool block)
817{
818 intel_de_rmw(display, PIPEDMC_BLOCK_PKGC_SW(pipe),
819 PIPEDMC_BLOCK_PKGC_SW_BLOCK_PKGC_ALWAYS, set: block ?
820 PIPEDMC_BLOCK_PKGC_SW_BLOCK_PKGC_ALWAYS : 0);
821}
822
823/**
824 * intel_dmc_start_pkgc_exit_at_start_of_undelayed_vblank() - start of PKG
825 * C-state exit
826 * @display: display instance
827 * @pipe: pipe which register use to block
828 * @enable: enable/disable
829 *
830 * This interface is target for Wa_16025596647 usage. I.e. start the package C
831 * exit at the start of the undelayed vblank
832 */
833void intel_dmc_start_pkgc_exit_at_start_of_undelayed_vblank(struct intel_display *display,
834 enum pipe pipe, bool enable)
835{
836 enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
837
838 dmc_configure_event(display, dmc_id, event_id: PIPEDMC_EVENT_VBLANK, enable);
839}
840
841/**
842 * intel_dmc_load_program() - write the firmware from memory to register.
843 * @display: display instance
844 *
845 * DMC firmware is read from a .bin file and kept in internal memory one time.
846 * Everytime display comes back from low power state this function is called to
847 * copy the firmware from internal memory to registers.
848 */
849void intel_dmc_load_program(struct intel_display *display)
850{
851 struct i915_power_domains *power_domains = &display->power.domains;
852 enum intel_dmc_id dmc_id;
853
854 if (!intel_dmc_has_payload(display))
855 return;
856
857 assert_display_rpm_held(display);
858
859 pipedmc_clock_gating_wa(display, enable: true);
860
861 for_each_dmc_id(dmc_id) {
862 dmc_load_program(display, dmc_id);
863 assert_dmc_loaded(display, dmc_id);
864 }
865
866 if (DISPLAY_VER(display) >= 20)
867 intel_de_write(display, DMC_FQ_W2_PTS_CFG_SEL,
868 PIPE_D_DMC_W2_PTS_CONFIG_SELECT(PIPE_D) |
869 PIPE_C_DMC_W2_PTS_CONFIG_SELECT(PIPE_C) |
870 PIPE_B_DMC_W2_PTS_CONFIG_SELECT(PIPE_B) |
871 PIPE_A_DMC_W2_PTS_CONFIG_SELECT(PIPE_A));
872
873 power_domains->dc_state = 0;
874
875 gen9_set_dc_state_debugmask(display);
876
877 pipedmc_clock_gating_wa(display, enable: false);
878}
879
880/**
881 * intel_dmc_disable_program() - disable the firmware
882 * @display: display instance
883 *
884 * Disable all event handlers in the firmware, making sure the firmware is
885 * inactive after the display is uninitialized.
886 */
887void intel_dmc_disable_program(struct intel_display *display)
888{
889 enum intel_dmc_id dmc_id;
890
891 if (!intel_dmc_has_payload(display))
892 return;
893
894 pipedmc_clock_gating_wa(display, enable: true);
895
896 for_each_dmc_id(dmc_id)
897 disable_all_event_handlers(display, dmc_id);
898
899 pipedmc_clock_gating_wa(display, enable: false);
900}
901
902static bool fw_info_matches_stepping(const struct intel_fw_info *fw_info,
903 const struct stepping_info *si)
904{
905 if ((fw_info->substepping == '*' && si->stepping == fw_info->stepping) ||
906 (si->stepping == fw_info->stepping && si->substepping == fw_info->substepping) ||
907 /*
908 * If we don't find a more specific one from above two checks, we
909 * then check for the generic one to be sure to work even with
910 * "broken firmware"
911 */
912 (si->stepping == '*' && si->substepping == fw_info->substepping) ||
913 (fw_info->stepping == '*' && fw_info->substepping == '*'))
914 return true;
915
916 return false;
917}
918
919/*
920 * Search fw_info table for dmc_offset to find firmware binary: num_entries is
921 * already sanitized.
922 */
923static void dmc_set_fw_offset(struct intel_dmc *dmc,
924 const struct intel_fw_info *fw_info,
925 unsigned int num_entries,
926 const struct stepping_info *si,
927 u8 package_ver)
928{
929 struct intel_display *display = dmc->display;
930 enum intel_dmc_id dmc_id;
931 unsigned int i;
932
933 for (i = 0; i < num_entries; i++) {
934 dmc_id = package_ver <= 1 ? DMC_FW_MAIN : fw_info[i].dmc_id;
935
936 if (!is_valid_dmc_id(dmc_id)) {
937 drm_dbg(display->drm, "Unsupported firmware id: %u\n", dmc_id);
938 continue;
939 }
940
941 /* More specific versions come first, so we don't even have to
942 * check for the stepping since we already found a previous FW
943 * for this id.
944 */
945 if (dmc->dmc_info[dmc_id].present)
946 continue;
947
948 if (fw_info_matches_stepping(fw_info: &fw_info[i], si)) {
949 dmc->dmc_info[dmc_id].present = true;
950 dmc->dmc_info[dmc_id].dmc_offset = fw_info[i].offset;
951 }
952 }
953}
954
955static bool dmc_mmio_addr_sanity_check(struct intel_dmc *dmc,
956 const u32 *mmioaddr, u32 mmio_count,
957 int header_ver, enum intel_dmc_id dmc_id)
958{
959 struct intel_display *display = dmc->display;
960 u32 start_range, end_range;
961 int i;
962
963 if (header_ver == 1) {
964 start_range = DMC_MMIO_START_RANGE;
965 end_range = DMC_MMIO_END_RANGE;
966 } else if (dmc_id == DMC_FW_MAIN) {
967 start_range = TGL_MAIN_MMIO_START;
968 end_range = TGL_MAIN_MMIO_END;
969 } else if (DISPLAY_VER(display) >= 13) {
970 start_range = ADLP_PIPE_MMIO_START;
971 end_range = ADLP_PIPE_MMIO_END;
972 } else if (DISPLAY_VER(display) >= 12) {
973 start_range = TGL_PIPE_MMIO_START(dmc_id);
974 end_range = TGL_PIPE_MMIO_END(dmc_id);
975 } else {
976 drm_warn(display->drm, "Unknown mmio range for sanity check");
977 return false;
978 }
979
980 for (i = 0; i < mmio_count; i++) {
981 if (mmioaddr[i] < start_range || mmioaddr[i] > end_range)
982 return false;
983 }
984
985 return true;
986}
987
988static u32 parse_dmc_fw_header(struct intel_dmc *dmc,
989 const struct intel_dmc_header_base *dmc_header,
990 size_t rem_size, enum intel_dmc_id dmc_id)
991{
992 struct intel_display *display = dmc->display;
993 struct dmc_fw_info *dmc_info = &dmc->dmc_info[dmc_id];
994 unsigned int header_len_bytes, dmc_header_size, payload_size, i;
995 const u32 *mmioaddr, *mmiodata;
996 u32 mmio_count, mmio_count_max, start_mmioaddr;
997 u8 *payload;
998
999 BUILD_BUG_ON(ARRAY_SIZE(dmc_info->mmioaddr) < DMC_V3_MAX_MMIO_COUNT ||
1000 ARRAY_SIZE(dmc_info->mmioaddr) < DMC_V1_MAX_MMIO_COUNT);
1001
1002 /*
1003 * Check if we can access common fields, we will checkc again below
1004 * after we have read the version
1005 */
1006 if (rem_size < sizeof(struct intel_dmc_header_base))
1007 goto error_truncated;
1008
1009 /* Cope with small differences between v1 and v3 */
1010 if (dmc_header->header_ver == 3) {
1011 const struct intel_dmc_header_v3 *v3 =
1012 (const struct intel_dmc_header_v3 *)dmc_header;
1013
1014 if (rem_size < sizeof(struct intel_dmc_header_v3))
1015 goto error_truncated;
1016
1017 mmioaddr = v3->mmioaddr;
1018 mmiodata = v3->mmiodata;
1019 mmio_count = v3->mmio_count;
1020 mmio_count_max = DMC_V3_MAX_MMIO_COUNT;
1021 /* header_len is in dwords */
1022 header_len_bytes = dmc_header->header_len * 4;
1023 start_mmioaddr = v3->start_mmioaddr;
1024 dmc_header_size = sizeof(*v3);
1025 } else if (dmc_header->header_ver == 1) {
1026 const struct intel_dmc_header_v1 *v1 =
1027 (const struct intel_dmc_header_v1 *)dmc_header;
1028
1029 if (rem_size < sizeof(struct intel_dmc_header_v1))
1030 goto error_truncated;
1031
1032 mmioaddr = v1->mmioaddr;
1033 mmiodata = v1->mmiodata;
1034 mmio_count = v1->mmio_count;
1035 mmio_count_max = DMC_V1_MAX_MMIO_COUNT;
1036 header_len_bytes = dmc_header->header_len;
1037 start_mmioaddr = DMC_V1_MMIO_START_RANGE;
1038 dmc_header_size = sizeof(*v1);
1039 } else {
1040 drm_err(display->drm, "Unknown DMC fw header version: %u\n",
1041 dmc_header->header_ver);
1042 return 0;
1043 }
1044
1045 if (header_len_bytes != dmc_header_size) {
1046 drm_err(display->drm, "DMC firmware has wrong dmc header length "
1047 "(%u bytes)\n", header_len_bytes);
1048 return 0;
1049 }
1050
1051 /* Cache the dmc header info. */
1052 if (mmio_count > mmio_count_max) {
1053 drm_err(display->drm, "DMC firmware has wrong mmio count %u\n", mmio_count);
1054 return 0;
1055 }
1056
1057 if (!dmc_mmio_addr_sanity_check(dmc, mmioaddr, mmio_count,
1058 header_ver: dmc_header->header_ver, dmc_id)) {
1059 drm_err(display->drm, "DMC firmware has Wrong MMIO Addresses\n");
1060 return 0;
1061 }
1062
1063 drm_dbg_kms(display->drm, "DMC %d:\n", dmc_id);
1064 for (i = 0; i < mmio_count; i++) {
1065 dmc_info->mmioaddr[i] = _MMIO(mmioaddr[i]);
1066 dmc_info->mmiodata[i] = mmiodata[i];
1067
1068 drm_dbg_kms(display->drm, " mmio[%d]: 0x%x = 0x%x%s%s\n",
1069 i, mmioaddr[i], mmiodata[i],
1070 is_dmc_evt_ctl_reg(display, dmc_id, dmc_info->mmioaddr[i]) ? " (EVT_CTL)" :
1071 is_dmc_evt_htp_reg(display, dmc_id, dmc_info->mmioaddr[i]) ? " (EVT_HTP)" : "",
1072 disable_dmc_evt(display, dmc_id, dmc_info->mmioaddr[i],
1073 dmc_info->mmiodata[i]) ? " (disabling)" : "");
1074 }
1075 dmc_info->mmio_count = mmio_count;
1076 dmc_info->start_mmioaddr = start_mmioaddr;
1077
1078 rem_size -= header_len_bytes;
1079
1080 /* fw_size is in dwords, so multiplied by 4 to convert into bytes. */
1081 payload_size = dmc_header->fw_size * 4;
1082 if (rem_size < payload_size)
1083 goto error_truncated;
1084
1085 if (payload_size > dmc->max_fw_size) {
1086 drm_err(display->drm, "DMC FW too big (%u bytes)\n", payload_size);
1087 return 0;
1088 }
1089 dmc_info->dmc_fw_size = dmc_header->fw_size;
1090
1091 dmc_info->payload = kmalloc(payload_size, GFP_KERNEL);
1092 if (!dmc_info->payload)
1093 return 0;
1094
1095 payload = (u8 *)(dmc_header) + header_len_bytes;
1096 memcpy(to: dmc_info->payload, from: payload, len: payload_size);
1097
1098 return header_len_bytes + payload_size;
1099
1100error_truncated:
1101 drm_err(display->drm, "Truncated DMC firmware, refusing.\n");
1102 return 0;
1103}
1104
1105static u32
1106parse_dmc_fw_package(struct intel_dmc *dmc,
1107 const struct intel_package_header *package_header,
1108 const struct stepping_info *si,
1109 size_t rem_size)
1110{
1111 struct intel_display *display = dmc->display;
1112 u32 package_size = sizeof(struct intel_package_header);
1113 u32 num_entries, max_entries;
1114 const struct intel_fw_info *fw_info;
1115
1116 if (rem_size < package_size)
1117 goto error_truncated;
1118
1119 if (package_header->header_ver == 1) {
1120 max_entries = PACKAGE_MAX_FW_INFO_ENTRIES;
1121 } else if (package_header->header_ver == 2) {
1122 max_entries = PACKAGE_V2_MAX_FW_INFO_ENTRIES;
1123 } else {
1124 drm_err(display->drm, "DMC firmware has unknown header version %u\n",
1125 package_header->header_ver);
1126 return 0;
1127 }
1128
1129 /*
1130 * We should always have space for max_entries,
1131 * even if not all are used
1132 */
1133 package_size += max_entries * sizeof(struct intel_fw_info);
1134 if (rem_size < package_size)
1135 goto error_truncated;
1136
1137 if (package_header->header_len * 4 != package_size) {
1138 drm_err(display->drm, "DMC firmware has wrong package header length "
1139 "(%u bytes)\n", package_size);
1140 return 0;
1141 }
1142
1143 num_entries = package_header->num_entries;
1144 if (WARN_ON(package_header->num_entries > max_entries))
1145 num_entries = max_entries;
1146
1147 fw_info = (const struct intel_fw_info *)
1148 ((u8 *)package_header + sizeof(*package_header));
1149 dmc_set_fw_offset(dmc, fw_info, num_entries, si,
1150 package_ver: package_header->header_ver);
1151
1152 /* dmc_offset is in dwords */
1153 return package_size;
1154
1155error_truncated:
1156 drm_err(display->drm, "Truncated DMC firmware, refusing.\n");
1157 return 0;
1158}
1159
1160/* Return number of bytes parsed or 0 on error */
1161static u32 parse_dmc_fw_css(struct intel_dmc *dmc,
1162 struct intel_css_header *css_header,
1163 size_t rem_size)
1164{
1165 struct intel_display *display = dmc->display;
1166
1167 if (rem_size < sizeof(struct intel_css_header)) {
1168 drm_err(display->drm, "Truncated DMC firmware, refusing.\n");
1169 return 0;
1170 }
1171
1172 if (sizeof(struct intel_css_header) !=
1173 (css_header->header_len * 4)) {
1174 drm_err(display->drm, "DMC firmware has wrong CSS header length "
1175 "(%u bytes)\n",
1176 (css_header->header_len * 4));
1177 return 0;
1178 }
1179
1180 dmc->version = css_header->version;
1181
1182 return sizeof(struct intel_css_header);
1183}
1184
1185static int parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
1186{
1187 struct intel_display *display = dmc->display;
1188 struct intel_css_header *css_header;
1189 struct intel_package_header *package_header;
1190 struct intel_dmc_header_base *dmc_header;
1191 struct stepping_info display_info = { '*', '*'};
1192 const struct stepping_info *si = intel_get_stepping_info(display, si: &display_info);
1193 enum intel_dmc_id dmc_id;
1194 u32 readcount = 0;
1195 u32 r, offset;
1196
1197 if (!fw)
1198 return -EINVAL;
1199
1200 /* Extract CSS Header information */
1201 css_header = (struct intel_css_header *)fw->data;
1202 r = parse_dmc_fw_css(dmc, css_header, rem_size: fw->size);
1203 if (!r)
1204 return -EINVAL;
1205
1206 readcount += r;
1207
1208 /* Extract Package Header information */
1209 package_header = (struct intel_package_header *)&fw->data[readcount];
1210 r = parse_dmc_fw_package(dmc, package_header, si, rem_size: fw->size - readcount);
1211 if (!r)
1212 return -EINVAL;
1213
1214 readcount += r;
1215
1216 for_each_dmc_id(dmc_id) {
1217 if (!dmc->dmc_info[dmc_id].present)
1218 continue;
1219
1220 offset = readcount + dmc->dmc_info[dmc_id].dmc_offset * 4;
1221 if (offset > fw->size) {
1222 drm_err(display->drm, "Reading beyond the fw_size\n");
1223 continue;
1224 }
1225
1226 dmc_header = (struct intel_dmc_header_base *)&fw->data[offset];
1227 parse_dmc_fw_header(dmc, dmc_header, rem_size: fw->size - offset, dmc_id);
1228 }
1229
1230 if (!intel_dmc_has_payload(display)) {
1231 drm_err(display->drm, "DMC firmware main program not found\n");
1232 return -ENOENT;
1233 }
1234
1235 return 0;
1236}
1237
1238static void intel_dmc_runtime_pm_get(struct intel_display *display)
1239{
1240 drm_WARN_ON(display->drm, display->dmc.wakeref);
1241 display->dmc.wakeref = intel_display_power_get(display, domain: POWER_DOMAIN_INIT);
1242}
1243
1244static void intel_dmc_runtime_pm_put(struct intel_display *display)
1245{
1246 intel_wakeref_t wakeref __maybe_unused =
1247 fetch_and_zero(&display->dmc.wakeref);
1248
1249 intel_display_power_put(display, domain: POWER_DOMAIN_INIT, wakeref);
1250}
1251
1252static const char *dmc_fallback_path(struct intel_display *display)
1253{
1254 if (display->platform.alderlake_p)
1255 return ADLP_DMC_FALLBACK_PATH;
1256
1257 return NULL;
1258}
1259
1260static void dmc_load_work_fn(struct work_struct *work)
1261{
1262 struct intel_dmc *dmc = container_of(work, typeof(*dmc), work);
1263 struct intel_display *display = dmc->display;
1264 const struct firmware *fw = NULL;
1265 const char *fallback_path;
1266 int err;
1267
1268 err = request_firmware(fw: &fw, name: dmc->fw_path, device: display->drm->dev);
1269
1270 if (err == -ENOENT && !dmc_firmware_param(display)) {
1271 fallback_path = dmc_fallback_path(display);
1272 if (fallback_path) {
1273 drm_dbg_kms(display->drm, "%s not found, falling back to %s\n",
1274 dmc->fw_path, fallback_path);
1275 err = request_firmware(fw: &fw, name: fallback_path, device: display->drm->dev);
1276 if (err == 0)
1277 dmc->fw_path = fallback_path;
1278 }
1279 }
1280
1281 if (err) {
1282 drm_notice(display->drm,
1283 "Failed to load DMC firmware %s (%pe). Disabling runtime power management.\n",
1284 dmc->fw_path, ERR_PTR(err));
1285 drm_notice(display->drm, "DMC firmware homepage: %s",
1286 INTEL_DMC_FIRMWARE_URL);
1287 return;
1288 }
1289
1290 err = parse_dmc_fw(dmc, fw);
1291 if (err) {
1292 drm_notice(display->drm,
1293 "Failed to parse DMC firmware %s (%pe). Disabling runtime power management.\n",
1294 dmc->fw_path, ERR_PTR(err));
1295 goto out;
1296 }
1297
1298 intel_dmc_load_program(display);
1299 intel_dmc_runtime_pm_put(display);
1300
1301 drm_info(display->drm, "Finished loading DMC firmware %s (v%u.%u)\n",
1302 dmc->fw_path, DMC_VERSION_MAJOR(dmc->version),
1303 DMC_VERSION_MINOR(dmc->version));
1304
1305out:
1306 release_firmware(fw);
1307}
1308
1309/**
1310 * intel_dmc_init() - initialize the firmware loading.
1311 * @display: display instance
1312 *
1313 * This function is called at the time of loading the display driver to read
1314 * firmware from a .bin file and copied into a internal memory.
1315 */
1316void intel_dmc_init(struct intel_display *display)
1317{
1318 struct intel_dmc *dmc;
1319
1320 if (!HAS_DMC(display))
1321 return;
1322
1323 /*
1324 * Obtain a runtime pm reference, until DMC is loaded, to avoid entering
1325 * runtime-suspend.
1326 *
1327 * On error, we return with the rpm wakeref held to prevent runtime
1328 * suspend as runtime suspend *requires* a working DMC for whatever
1329 * reason.
1330 */
1331 intel_dmc_runtime_pm_get(display);
1332
1333 dmc = kzalloc(sizeof(*dmc), GFP_KERNEL);
1334 if (!dmc)
1335 return;
1336
1337 dmc->display = display;
1338
1339 INIT_WORK(&dmc->work, dmc_load_work_fn);
1340
1341 dmc->fw_path = dmc_firmware_default(display, size: &dmc->max_fw_size);
1342
1343 if (dmc_firmware_param_disabled(display)) {
1344 drm_info(display->drm, "Disabling DMC firmware and runtime PM\n");
1345 goto out;
1346 }
1347
1348 if (dmc_firmware_param(display))
1349 dmc->fw_path = dmc_firmware_param(display);
1350
1351 if (!dmc->fw_path) {
1352 drm_dbg_kms(display->drm,
1353 "No known DMC firmware for platform, disabling runtime PM\n");
1354 goto out;
1355 }
1356
1357 display->dmc.dmc = dmc;
1358
1359 drm_dbg_kms(display->drm, "Loading %s\n", dmc->fw_path);
1360 queue_work(wq: display->wq.unordered, work: &dmc->work);
1361
1362 return;
1363
1364out:
1365 kfree(objp: dmc);
1366}
1367
1368/**
1369 * intel_dmc_suspend() - prepare DMC firmware before system suspend
1370 * @display: display instance
1371 *
1372 * Prepare the DMC firmware before entering system suspend. This includes
1373 * flushing pending work items and releasing any resources acquired during
1374 * init.
1375 */
1376void intel_dmc_suspend(struct intel_display *display)
1377{
1378 struct intel_dmc *dmc = display_to_dmc(display);
1379
1380 if (!HAS_DMC(display))
1381 return;
1382
1383 if (dmc)
1384 flush_work(work: &dmc->work);
1385
1386 /* Drop the reference held in case DMC isn't loaded. */
1387 if (!intel_dmc_has_payload(display))
1388 intel_dmc_runtime_pm_put(display);
1389}
1390
1391void intel_dmc_wait_fw_load(struct intel_display *display)
1392{
1393 struct intel_dmc *dmc = display_to_dmc(display);
1394
1395 if (!HAS_DMC(display))
1396 return;
1397
1398 if (dmc)
1399 flush_work(work: &dmc->work);
1400}
1401
1402/**
1403 * intel_dmc_resume() - init DMC firmware during system resume
1404 * @display: display instance
1405 *
1406 * Reinitialize the DMC firmware during system resume, reacquiring any
1407 * resources released in intel_dmc_suspend().
1408 */
1409void intel_dmc_resume(struct intel_display *display)
1410{
1411 if (!HAS_DMC(display))
1412 return;
1413
1414 /*
1415 * Reacquire the reference to keep RPM disabled in case DMC isn't
1416 * loaded.
1417 */
1418 if (!intel_dmc_has_payload(display))
1419 intel_dmc_runtime_pm_get(display);
1420}
1421
1422/**
1423 * intel_dmc_fini() - unload the DMC firmware.
1424 * @display: display instance
1425 *
1426 * Firmmware unloading includes freeing the internal memory and reset the
1427 * firmware loading status.
1428 */
1429void intel_dmc_fini(struct intel_display *display)
1430{
1431 struct intel_dmc *dmc = display_to_dmc(display);
1432 enum intel_dmc_id dmc_id;
1433
1434 if (!HAS_DMC(display))
1435 return;
1436
1437 intel_dmc_suspend(display);
1438 drm_WARN_ON(display->drm, display->dmc.wakeref);
1439
1440 if (dmc) {
1441 for_each_dmc_id(dmc_id)
1442 kfree(objp: dmc->dmc_info[dmc_id].payload);
1443
1444 kfree(objp: dmc);
1445 display->dmc.dmc = NULL;
1446 }
1447}
1448
1449struct intel_dmc_snapshot {
1450 bool initialized;
1451 bool loaded;
1452 u32 version;
1453};
1454
1455struct intel_dmc_snapshot *intel_dmc_snapshot_capture(struct intel_display *display)
1456{
1457 struct intel_dmc *dmc = display_to_dmc(display);
1458 struct intel_dmc_snapshot *snapshot;
1459
1460 if (!HAS_DMC(display))
1461 return NULL;
1462
1463 snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC);
1464 if (!snapshot)
1465 return NULL;
1466
1467 snapshot->initialized = dmc;
1468 snapshot->loaded = intel_dmc_has_payload(display);
1469 if (dmc)
1470 snapshot->version = dmc->version;
1471
1472 return snapshot;
1473}
1474
1475void intel_dmc_snapshot_print(const struct intel_dmc_snapshot *snapshot, struct drm_printer *p)
1476{
1477 if (!snapshot)
1478 return;
1479
1480 drm_printf(p, f: "DMC initialized: %s\n", str_yes_no(v: snapshot->initialized));
1481 drm_printf(p, f: "DMC loaded: %s\n", str_yes_no(v: snapshot->loaded));
1482 if (snapshot->initialized)
1483 drm_printf(p, f: "DMC fw version: %d.%d\n",
1484 DMC_VERSION_MAJOR(snapshot->version),
1485 DMC_VERSION_MINOR(snapshot->version));
1486}
1487
1488void intel_dmc_update_dc6_allowed_count(struct intel_display *display,
1489 bool start_tracking)
1490{
1491 struct intel_dmc *dmc = display_to_dmc(display);
1492 u32 dc5_cur_count;
1493
1494 if (DISPLAY_VER(dmc->display) < 14)
1495 return;
1496
1497 dc5_cur_count = intel_de_read(display: dmc->display, DG1_DMC_DEBUG_DC5_COUNT);
1498
1499 if (!start_tracking)
1500 dmc->dc6_allowed.count += dc5_cur_count - dmc->dc6_allowed.dc5_start;
1501
1502 dmc->dc6_allowed.dc5_start = dc5_cur_count;
1503}
1504
1505static bool intel_dmc_get_dc6_allowed_count(struct intel_display *display, u32 *count)
1506{
1507 struct i915_power_domains *power_domains = &display->power.domains;
1508 struct intel_dmc *dmc = display_to_dmc(display);
1509 bool dc6_enabled;
1510
1511 if (DISPLAY_VER(display) < 14)
1512 return false;
1513
1514 mutex_lock(lock: &power_domains->lock);
1515 dc6_enabled = intel_de_read(display, DC_STATE_EN) &
1516 DC_STATE_EN_UPTO_DC6;
1517 if (dc6_enabled)
1518 intel_dmc_update_dc6_allowed_count(display, start_tracking: false);
1519
1520 *count = dmc->dc6_allowed.count;
1521 mutex_unlock(lock: &power_domains->lock);
1522
1523 return true;
1524}
1525
1526static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
1527{
1528 struct intel_display *display = m->private;
1529 struct intel_dmc *dmc = display_to_dmc(display);
1530 struct ref_tracker *wakeref;
1531 i915_reg_t dc5_reg, dc6_reg = INVALID_MMIO_REG;
1532 u32 dc6_allowed_count;
1533
1534 if (!HAS_DMC(display))
1535 return -ENODEV;
1536
1537 wakeref = intel_display_rpm_get(display);
1538
1539 seq_printf(m, fmt: "DMC initialized: %s\n", str_yes_no(v: dmc));
1540 seq_printf(m, fmt: "fw loaded: %s\n",
1541 str_yes_no(v: intel_dmc_has_payload(display)));
1542 seq_printf(m, fmt: "path: %s\n", dmc ? dmc->fw_path : "N/A");
1543 seq_printf(m, fmt: "Pipe A fw needed: %s\n",
1544 str_yes_no(DISPLAY_VER(display) >= 12));
1545 seq_printf(m, fmt: "Pipe A fw loaded: %s\n",
1546 str_yes_no(v: has_dmc_id_fw(display, dmc_id: DMC_FW_PIPEA)));
1547 seq_printf(m, fmt: "Pipe B fw needed: %s\n",
1548 str_yes_no(v: display->platform.alderlake_p ||
1549 DISPLAY_VER(display) >= 14));
1550 seq_printf(m, fmt: "Pipe B fw loaded: %s\n",
1551 str_yes_no(v: has_dmc_id_fw(display, dmc_id: DMC_FW_PIPEB)));
1552
1553 if (!intel_dmc_has_payload(display))
1554 goto out;
1555
1556 seq_printf(m, fmt: "version: %d.%d\n", DMC_VERSION_MAJOR(dmc->version),
1557 DMC_VERSION_MINOR(dmc->version));
1558
1559 if (DISPLAY_VER(display) >= 12) {
1560 i915_reg_t dc3co_reg;
1561
1562 if (display->platform.dgfx || DISPLAY_VER(display) >= 14) {
1563 dc3co_reg = DG1_DMC_DEBUG3;
1564 dc5_reg = DG1_DMC_DEBUG_DC5_COUNT;
1565 } else {
1566 dc3co_reg = TGL_DMC_DEBUG3;
1567 dc5_reg = TGL_DMC_DEBUG_DC5_COUNT;
1568 dc6_reg = TGL_DMC_DEBUG_DC6_COUNT;
1569 }
1570
1571 seq_printf(m, fmt: "DC3CO count: %d\n",
1572 intel_de_read(display, reg: dc3co_reg));
1573 } else {
1574 dc5_reg = display->platform.broxton ? BXT_DMC_DC3_DC5_COUNT :
1575 SKL_DMC_DC3_DC5_COUNT;
1576 if (!display->platform.geminilake && !display->platform.broxton)
1577 dc6_reg = SKL_DMC_DC5_DC6_COUNT;
1578 }
1579
1580 seq_printf(m, fmt: "DC3 -> DC5 count: %d\n", intel_de_read(display, reg: dc5_reg));
1581
1582 if (intel_dmc_get_dc6_allowed_count(display, count: &dc6_allowed_count))
1583 seq_printf(m, fmt: "DC5 -> DC6 allowed count: %d\n",
1584 dc6_allowed_count);
1585 else if (i915_mmio_reg_valid(dc6_reg))
1586 seq_printf(m, fmt: "DC5 -> DC6 count: %d\n",
1587 intel_de_read(display, reg: dc6_reg));
1588
1589 seq_printf(m, fmt: "program base: 0x%08x\n",
1590 intel_de_read(display, DMC_PROGRAM(dmc->dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)));
1591
1592out:
1593 seq_printf(m, fmt: "ssp base: 0x%08x\n",
1594 intel_de_read(display, DMC_SSP_BASE));
1595 seq_printf(m, fmt: "htp: 0x%08x\n", intel_de_read(display, DMC_HTP_SKL));
1596
1597 intel_display_rpm_put(display, wakeref);
1598
1599 return 0;
1600}
1601
1602DEFINE_SHOW_ATTRIBUTE(intel_dmc_debugfs_status);
1603
1604void intel_dmc_debugfs_register(struct intel_display *display)
1605{
1606 debugfs_create_file("i915_dmc_info", 0444, display->drm->debugfs_root,
1607 display, &intel_dmc_debugfs_status_fops);
1608}
1609
1610void intel_pipedmc_irq_handler(struct intel_display *display, enum pipe pipe)
1611{
1612 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
1613 u32 tmp = 0, int_vector;
1614
1615 if (DISPLAY_VER(display) >= 20) {
1616 tmp = intel_de_read(display, PIPEDMC_INTERRUPT(pipe));
1617 intel_de_write(display, PIPEDMC_INTERRUPT(pipe), val: tmp);
1618
1619 if (tmp & PIPEDMC_FLIPQ_PROG_DONE) {
1620 spin_lock(lock: &display->drm->event_lock);
1621
1622 if (crtc->flipq_event) {
1623 /*
1624 * Update vblank counter/timestamp in case it
1625 * hasn't been done yet for this frame.
1626 */
1627 drm_crtc_accurate_vblank_count(crtc: &crtc->base);
1628
1629 drm_crtc_send_vblank_event(crtc: &crtc->base, e: crtc->flipq_event);
1630 crtc->flipq_event = NULL;
1631 }
1632
1633 spin_unlock(lock: &display->drm->event_lock);
1634 }
1635
1636 if (tmp & PIPEDMC_ATS_FAULT)
1637 drm_err_ratelimited(display->drm, "[CRTC:%d:%s] PIPEDMC ATS fault\n",
1638 crtc->base.base.id, crtc->base.name);
1639 if (tmp & PIPEDMC_GTT_FAULT)
1640 drm_err_ratelimited(display->drm, "[CRTC:%d:%s] PIPEDMC GTT fault\n",
1641 crtc->base.base.id, crtc->base.name);
1642 if (tmp & PIPEDMC_ERROR)
1643 drm_err(display->drm, "[CRTC:%d:%s]] PIPEDMC error\n",
1644 crtc->base.base.id, crtc->base.name);
1645 }
1646
1647 int_vector = intel_de_read(display, PIPEDMC_STATUS(pipe)) & PIPEDMC_INT_VECTOR_MASK;
1648 if (tmp == 0 && int_vector != 0)
1649 drm_err(display->drm, "[CRTC:%d:%s]] PIPEDMC interrupt vector 0x%x\n",
1650 crtc->base.base.id, crtc->base.name, tmp);
1651}
1652
1653void intel_pipedmc_enable_event(struct intel_crtc *crtc,
1654 enum pipedmc_event_id event)
1655{
1656 struct intel_display *display = to_intel_display(crtc);
1657 enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(crtc->pipe);
1658
1659 dmc_configure_event(display, dmc_id, event_id: event, enable: true);
1660}
1661
1662void intel_pipedmc_disable_event(struct intel_crtc *crtc,
1663 enum pipedmc_event_id event)
1664{
1665 struct intel_display *display = to_intel_display(crtc);
1666 enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(crtc->pipe);
1667
1668 dmc_configure_event(display, dmc_id, event_id: event, enable: false);
1669}
1670
1671u32 intel_pipedmc_start_mmioaddr(struct intel_crtc *crtc)
1672{
1673 struct intel_display *display = to_intel_display(crtc);
1674 struct intel_dmc *dmc = display_to_dmc(display);
1675 enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(crtc->pipe);
1676
1677 return dmc ? dmc->dmc_info[dmc_id].start_mmioaddr : 0;
1678}
1679