1/*
2 * Copyright (c) 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Keith Packard <keithp@keithp.com>
26 * Mika Kuoppala <mika.kuoppala@intel.com>
27 *
28 */
29
30#include <linux/ascii85.h>
31#include <linux/debugfs.h>
32#include <linux/highmem.h>
33#include <linux/nmi.h>
34#include <linux/pagevec.h>
35#include <linux/scatterlist.h>
36#include <linux/string_helpers.h>
37#include <linux/utsname.h>
38#include <linux/zlib.h>
39
40#include <drm/drm_cache.h>
41#include <drm/drm_print.h>
42
43#include "display/intel_display_snapshot.h"
44
45#include "gem/i915_gem_context.h"
46#include "gem/i915_gem_lmem.h"
47#include "gt/intel_engine_regs.h"
48#include "gt/intel_gt.h"
49#include "gt/intel_gt_mcr.h"
50#include "gt/intel_gt_pm.h"
51#include "gt/intel_gt_regs.h"
52#include "gt/uc/intel_guc_capture.h"
53
54#include "i915_driver.h"
55#include "i915_drv.h"
56#include "i915_gpu_error.h"
57#include "i915_memcpy.h"
58#include "i915_reg.h"
59#include "i915_scatterlist.h"
60#include "i915_sysfs.h"
61#include "i915_utils.h"
62
63#define ALLOW_FAIL (__GFP_KSWAPD_RECLAIM | __GFP_RETRY_MAYFAIL | __GFP_NOWARN)
64#define ATOMIC_MAYFAIL (GFP_ATOMIC | __GFP_NOWARN)
65
66static void __sg_set_buf(struct scatterlist *sg,
67 void *addr, unsigned int len, loff_t it)
68{
69 sg->page_link = (unsigned long)virt_to_page(addr);
70 sg->offset = offset_in_page(addr);
71 sg->length = len;
72 sg->dma_address = it;
73}
74
75static bool __i915_error_grow(struct drm_i915_error_state_buf *e, size_t len)
76{
77 if (!len)
78 return false;
79
80 if (e->bytes + len + 1 <= e->size)
81 return true;
82
83 if (e->bytes) {
84 __sg_set_buf(sg: e->cur++, addr: e->buf, len: e->bytes, it: e->iter);
85 e->iter += e->bytes;
86 e->buf = NULL;
87 e->bytes = 0;
88 }
89
90 if (e->cur == e->end) {
91 struct scatterlist *sgl;
92
93 sgl = (typeof(sgl))__get_free_page(ALLOW_FAIL);
94 if (!sgl) {
95 e->err = -ENOMEM;
96 return false;
97 }
98
99 if (e->cur) {
100 e->cur->offset = 0;
101 e->cur->length = 0;
102 e->cur->page_link =
103 (unsigned long)sgl | SG_CHAIN;
104 } else {
105 e->sgl = sgl;
106 }
107
108 e->cur = sgl;
109 e->end = sgl + SG_MAX_SINGLE_ALLOC - 1;
110 }
111
112 e->size = ALIGN(len + 1, SZ_64K);
113 e->buf = kmalloc(e->size, ALLOW_FAIL);
114 if (!e->buf) {
115 e->size = PAGE_ALIGN(len + 1);
116 e->buf = kmalloc(e->size, GFP_KERNEL);
117 }
118 if (!e->buf) {
119 e->err = -ENOMEM;
120 return false;
121 }
122
123 return true;
124}
125
126__printf(2, 0)
127static void i915_error_vprintf(struct drm_i915_error_state_buf *e,
128 const char *fmt, va_list args)
129{
130 va_list ap;
131 int len;
132
133 if (e->err)
134 return;
135
136 va_copy(ap, args);
137 len = vsnprintf(NULL, size: 0, fmt, args: ap);
138 va_end(ap);
139 if (len <= 0) {
140 e->err = len;
141 return;
142 }
143
144 if (!__i915_error_grow(e, len))
145 return;
146
147 GEM_BUG_ON(e->bytes >= e->size);
148 len = vscnprintf(buf: e->buf + e->bytes, size: e->size - e->bytes, fmt, args);
149 if (len < 0) {
150 e->err = len;
151 return;
152 }
153 e->bytes += len;
154}
155
156static void i915_error_puts(struct drm_i915_error_state_buf *e, const char *str)
157{
158 unsigned len;
159
160 if (e->err || !str)
161 return;
162
163 len = strlen(str);
164 if (!__i915_error_grow(e, len))
165 return;
166
167 GEM_BUG_ON(e->bytes + len > e->size);
168 memcpy(to: e->buf + e->bytes, from: str, len);
169 e->bytes += len;
170}
171
172#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
173#define err_puts(e, s) i915_error_puts(e, s)
174
175static void __i915_printfn_error(struct drm_printer *p, struct va_format *vaf)
176{
177 i915_error_vprintf(e: p->arg, fmt: vaf->fmt, args: *vaf->va);
178}
179
180static inline struct drm_printer
181i915_error_printer(struct drm_i915_error_state_buf *e)
182{
183 struct drm_printer p = {
184 .printfn = __i915_printfn_error,
185 .arg = e,
186 };
187 return p;
188}
189
190/* single threaded page allocator with a reserved stash for emergencies */
191static void pool_fini(struct folio_batch *fbatch)
192{
193 folio_batch_release(fbatch);
194}
195
196static int pool_refill(struct folio_batch *fbatch, gfp_t gfp)
197{
198 while (folio_batch_space(fbatch)) {
199 struct folio *folio;
200
201 folio = folio_alloc(gfp, 0);
202 if (!folio)
203 return -ENOMEM;
204
205 folio_batch_add(fbatch, folio);
206 }
207
208 return 0;
209}
210
211static int pool_init(struct folio_batch *fbatch, gfp_t gfp)
212{
213 int err;
214
215 folio_batch_init(fbatch);
216
217 err = pool_refill(fbatch, gfp);
218 if (err)
219 pool_fini(fbatch);
220
221 return err;
222}
223
224static void *pool_alloc(struct folio_batch *fbatch, gfp_t gfp)
225{
226 struct folio *folio;
227
228 folio = folio_alloc(gfp, 0);
229 if (!folio && folio_batch_count(fbatch))
230 folio = fbatch->folios[--fbatch->nr];
231
232 return folio ? folio_address(folio) : NULL;
233}
234
235static void pool_free(struct folio_batch *fbatch, void *addr)
236{
237 struct folio *folio = virt_to_folio(x: addr);
238
239 if (folio_batch_space(fbatch))
240 folio_batch_add(fbatch, folio);
241 else
242 folio_put(folio);
243}
244
245#ifdef CONFIG_DRM_I915_COMPRESS_ERROR
246
247struct i915_vma_compress {
248 struct folio_batch pool;
249 struct z_stream_s zstream;
250 void *tmp;
251};
252
253static bool compress_init(struct i915_vma_compress *c)
254{
255 struct z_stream_s *zstream = &c->zstream;
256
257 if (pool_init(fbatch: &c->pool, ALLOW_FAIL))
258 return false;
259
260 zstream->workspace =
261 kmalloc(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
262 ALLOW_FAIL);
263 if (!zstream->workspace) {
264 pool_fini(fbatch: &c->pool);
265 return false;
266 }
267
268 c->tmp = NULL;
269 if (i915_has_memcpy_from_wc())
270 c->tmp = pool_alloc(fbatch: &c->pool, ALLOW_FAIL);
271
272 return true;
273}
274
275static bool compress_start(struct i915_vma_compress *c)
276{
277 struct z_stream_s *zstream = &c->zstream;
278 void *workspace = zstream->workspace;
279
280 memset(s: zstream, c: 0, n: sizeof(*zstream));
281 zstream->workspace = workspace;
282
283 return zlib_deflateInit(zstream, Z_DEFAULT_COMPRESSION) == Z_OK;
284}
285
286static void *compress_next_page(struct i915_vma_compress *c,
287 struct i915_vma_coredump *dst)
288{
289 void *page_addr;
290 struct page *page;
291
292 page_addr = pool_alloc(fbatch: &c->pool, ALLOW_FAIL);
293 if (!page_addr)
294 return ERR_PTR(error: -ENOMEM);
295
296 page = virt_to_page(page_addr);
297 list_add_tail(new: &page->lru, head: &dst->page_list);
298 return page_addr;
299}
300
301static int compress_page(struct i915_vma_compress *c,
302 void *src,
303 struct i915_vma_coredump *dst,
304 bool wc)
305{
306 struct z_stream_s *zstream = &c->zstream;
307
308 zstream->next_in = src;
309 if (wc && c->tmp && i915_memcpy_from_wc(dst: c->tmp, src, PAGE_SIZE))
310 zstream->next_in = c->tmp;
311 zstream->avail_in = PAGE_SIZE;
312
313 do {
314 if (zstream->avail_out == 0) {
315 zstream->next_out = compress_next_page(c, dst);
316 if (IS_ERR(ptr: zstream->next_out))
317 return PTR_ERR(ptr: zstream->next_out);
318
319 zstream->avail_out = PAGE_SIZE;
320 }
321
322 if (zlib_deflate(strm: zstream, Z_NO_FLUSH) != Z_OK)
323 return -EIO;
324
325 cond_resched();
326 } while (zstream->avail_in);
327
328 /* Fallback to uncompressed if we increase size? */
329 if (0 && zstream->total_out > zstream->total_in)
330 return -E2BIG;
331
332 return 0;
333}
334
335static int compress_flush(struct i915_vma_compress *c,
336 struct i915_vma_coredump *dst)
337{
338 struct z_stream_s *zstream = &c->zstream;
339
340 do {
341 switch (zlib_deflate(strm: zstream, Z_FINISH)) {
342 case Z_OK: /* more space requested */
343 zstream->next_out = compress_next_page(c, dst);
344 if (IS_ERR(ptr: zstream->next_out))
345 return PTR_ERR(ptr: zstream->next_out);
346
347 zstream->avail_out = PAGE_SIZE;
348 break;
349
350 case Z_STREAM_END:
351 goto end;
352
353 default: /* any error */
354 return -EIO;
355 }
356 } while (1);
357
358end:
359 memset(s: zstream->next_out, c: 0, n: zstream->avail_out);
360 dst->unused = zstream->avail_out;
361 return 0;
362}
363
364static void compress_finish(struct i915_vma_compress *c)
365{
366 zlib_deflateEnd(strm: &c->zstream);
367}
368
369static void compress_fini(struct i915_vma_compress *c)
370{
371 kfree(objp: c->zstream.workspace);
372 if (c->tmp)
373 pool_free(fbatch: &c->pool, addr: c->tmp);
374 pool_fini(fbatch: &c->pool);
375}
376
377static void err_compression_marker(struct drm_i915_error_state_buf *m)
378{
379 err_puts(m, ":");
380}
381
382#else
383
384struct i915_vma_compress {
385 struct folio_batch pool;
386};
387
388static bool compress_init(struct i915_vma_compress *c)
389{
390 return pool_init(&c->pool, ALLOW_FAIL) == 0;
391}
392
393static bool compress_start(struct i915_vma_compress *c)
394{
395 return true;
396}
397
398static int compress_page(struct i915_vma_compress *c,
399 void *src,
400 struct i915_vma_coredump *dst,
401 bool wc)
402{
403 void *ptr;
404
405 ptr = pool_alloc(&c->pool, ALLOW_FAIL);
406 if (!ptr)
407 return -ENOMEM;
408
409 if (!(wc && i915_memcpy_from_wc(ptr, src, PAGE_SIZE)))
410 memcpy(ptr, src, PAGE_SIZE);
411 list_add_tail(&virt_to_page(ptr)->lru, &dst->page_list);
412 cond_resched();
413
414 return 0;
415}
416
417static int compress_flush(struct i915_vma_compress *c,
418 struct i915_vma_coredump *dst)
419{
420 return 0;
421}
422
423static void compress_finish(struct i915_vma_compress *c)
424{
425}
426
427static void compress_fini(struct i915_vma_compress *c)
428{
429 pool_fini(&c->pool);
430}
431
432static void err_compression_marker(struct drm_i915_error_state_buf *m)
433{
434 err_puts(m, "~");
435}
436
437#endif
438
439static void error_print_instdone(struct drm_i915_error_state_buf *m,
440 const struct intel_engine_coredump *ee)
441{
442 int slice;
443 int subslice;
444 int iter;
445
446 err_printf(m, " INSTDONE: 0x%08x\n",
447 ee->instdone.instdone);
448
449 if (ee->engine->class != RENDER_CLASS || GRAPHICS_VER(m->i915) <= 3)
450 return;
451
452 err_printf(m, " SC_INSTDONE: 0x%08x\n",
453 ee->instdone.slice_common);
454
455 if (GRAPHICS_VER(m->i915) <= 6)
456 return;
457
458 for_each_ss_steering(iter, ee->engine->gt, slice, subslice)
459 err_printf(m, " SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
460 slice, subslice,
461 ee->instdone.sampler[slice][subslice]);
462
463 for_each_ss_steering(iter, ee->engine->gt, slice, subslice)
464 err_printf(m, " ROW_INSTDONE[%d][%d]: 0x%08x\n",
465 slice, subslice,
466 ee->instdone.row[slice][subslice]);
467
468 if (GRAPHICS_VER(m->i915) < 12)
469 return;
470
471 if (GRAPHICS_VER_FULL(m->i915) >= IP_VER(12, 55)) {
472 for_each_ss_steering(iter, ee->engine->gt, slice, subslice)
473 err_printf(m, " GEOM_SVGUNIT_INSTDONE[%d][%d]: 0x%08x\n",
474 slice, subslice,
475 ee->instdone.geom_svg[slice][subslice]);
476 }
477
478 err_printf(m, " SC_INSTDONE_EXTRA: 0x%08x\n",
479 ee->instdone.slice_common_extra[0]);
480 err_printf(m, " SC_INSTDONE_EXTRA2: 0x%08x\n",
481 ee->instdone.slice_common_extra[1]);
482}
483
484static void error_print_request(struct drm_i915_error_state_buf *m,
485 const char *prefix,
486 const struct i915_request_coredump *erq)
487{
488 if (!erq->seqno)
489 return;
490
491 err_printf(m, "%s pid %d, seqno %8x:%08x%s%s, prio %d, head %08x, tail %08x\n",
492 prefix, erq->pid, erq->context, erq->seqno,
493 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
494 &erq->flags) ? "!" : "",
495 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
496 &erq->flags) ? "+" : "",
497 erq->sched_attr.priority,
498 erq->head, erq->tail);
499}
500
501static void error_print_context(struct drm_i915_error_state_buf *m,
502 const char *header,
503 const struct i915_gem_context_coredump *ctx)
504{
505 err_printf(m, "%s%s[%d] prio %d, guilty %d active %d, runtime total %lluns, avg %lluns\n",
506 header, ctx->comm, ctx->pid, ctx->sched_attr.priority,
507 ctx->guilty, ctx->active,
508 ctx->total_runtime, ctx->avg_runtime);
509 err_printf(m, " context timeline seqno %u\n", ctx->hwsp_seqno);
510}
511
512static struct i915_vma_coredump *
513__find_vma(struct i915_vma_coredump *vma, const char *name)
514{
515 while (vma) {
516 if (strcmp(vma->name, name) == 0)
517 return vma;
518 vma = vma->next;
519 }
520
521 return NULL;
522}
523
524static struct i915_vma_coredump *
525intel_gpu_error_find_batch(const struct intel_engine_coredump *ee)
526{
527 return __find_vma(vma: ee->vma, name: "batch");
528}
529
530static void error_print_engine(struct drm_i915_error_state_buf *m,
531 const struct intel_engine_coredump *ee)
532{
533 struct i915_vma_coredump *batch;
534 int n;
535
536 err_printf(m, "%s command stream:\n", ee->engine->name);
537 err_printf(m, " CCID: 0x%08x\n", ee->ccid);
538 err_printf(m, " START: 0x%08x\n", ee->start);
539 err_printf(m, " HEAD: 0x%08x [0x%08x]\n", ee->head, ee->rq_head);
540 err_printf(m, " TAIL: 0x%08x [0x%08x, 0x%08x]\n",
541 ee->tail, ee->rq_post, ee->rq_tail);
542 err_printf(m, " CTL: 0x%08x\n", ee->ctl);
543 err_printf(m, " MODE: 0x%08x\n", ee->mode);
544 err_printf(m, " HWS: 0x%08x\n", ee->hws);
545 err_printf(m, " ACTHD: 0x%08x %08x\n",
546 (u32)(ee->acthd>>32), (u32)ee->acthd);
547 err_printf(m, " IPEIR: 0x%08x\n", ee->ipeir);
548 err_printf(m, " IPEHR: 0x%08x\n", ee->ipehr);
549 err_printf(m, " ESR: 0x%08x\n", ee->esr);
550
551 error_print_instdone(m, ee);
552
553 batch = intel_gpu_error_find_batch(ee);
554 if (batch) {
555 u64 start = batch->gtt_offset;
556 u64 end = start + batch->gtt_size;
557
558 err_printf(m, " batch: [0x%08x_%08x, 0x%08x_%08x]\n",
559 upper_32_bits(start), lower_32_bits(start),
560 upper_32_bits(end), lower_32_bits(end));
561 }
562 if (GRAPHICS_VER(m->i915) >= 4) {
563 err_printf(m, " BBADDR: 0x%08x_%08x\n",
564 (u32)(ee->bbaddr>>32), (u32)ee->bbaddr);
565 err_printf(m, " BB_STATE: 0x%08x\n", ee->bbstate);
566 err_printf(m, " INSTPS: 0x%08x\n", ee->instps);
567 }
568 err_printf(m, " INSTPM: 0x%08x\n", ee->instpm);
569 err_printf(m, " FADDR: 0x%08x %08x\n", upper_32_bits(ee->faddr),
570 lower_32_bits(ee->faddr));
571 if (GRAPHICS_VER(m->i915) >= 6) {
572 err_printf(m, " RC PSMI: 0x%08x\n", ee->rc_psmi);
573 err_printf(m, " FAULT_REG: 0x%08x\n", ee->fault_reg);
574 }
575 if (GRAPHICS_VER(m->i915) >= 11) {
576 err_printf(m, " NOPID: 0x%08x\n", ee->nopid);
577 err_printf(m, " EXCC: 0x%08x\n", ee->excc);
578 err_printf(m, " CMD_CCTL: 0x%08x\n", ee->cmd_cctl);
579 err_printf(m, " CSCMDOP: 0x%08x\n", ee->cscmdop);
580 err_printf(m, " CTX_SR_CTL: 0x%08x\n", ee->ctx_sr_ctl);
581 err_printf(m, " DMA_FADDR_HI: 0x%08x\n", ee->dma_faddr_hi);
582 err_printf(m, " DMA_FADDR_LO: 0x%08x\n", ee->dma_faddr_lo);
583 }
584 if (HAS_PPGTT(m->i915)) {
585 err_printf(m, " GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode);
586
587 if (GRAPHICS_VER(m->i915) >= 8) {
588 int i;
589 for (i = 0; i < 4; i++)
590 err_printf(m, " PDP%d: 0x%016llx\n",
591 i, ee->vm_info.pdp[i]);
592 } else {
593 err_printf(m, " PP_DIR_BASE: 0x%08x\n",
594 ee->vm_info.pp_dir_base);
595 }
596 }
597
598 for (n = 0; n < ee->num_ports; n++) {
599 err_printf(m, " ELSP[%d]:", n);
600 error_print_request(m, prefix: " ", erq: &ee->execlist[n]);
601 }
602}
603
604void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
605{
606 va_list args;
607
608 va_start(args, f);
609 i915_error_vprintf(e, fmt: f, args);
610 va_end(args);
611}
612
613static void intel_gpu_error_print_vma(struct drm_i915_error_state_buf *m,
614 const struct intel_engine_cs *engine,
615 const struct i915_vma_coredump *vma)
616{
617 char out[ASCII85_BUFSZ];
618 struct page *page;
619
620 if (!vma)
621 return;
622
623 err_printf(m, "%s --- %s = 0x%08x %08x\n",
624 engine ? engine->name : "global", vma->name,
625 upper_32_bits(vma->gtt_offset),
626 lower_32_bits(vma->gtt_offset));
627
628 if (vma->gtt_page_sizes > I915_GTT_PAGE_SIZE_4K)
629 err_printf(m, "gtt_page_sizes = 0x%08x\n", vma->gtt_page_sizes);
630
631 err_compression_marker(m);
632 list_for_each_entry(page, &vma->page_list, lru) {
633 int i, len;
634 const u32 *addr = page_address(page);
635
636 len = PAGE_SIZE;
637 if (page == list_last_entry(&vma->page_list, typeof(*page), lru))
638 len -= vma->unused;
639 len = ascii85_encode_len(len);
640
641 for (i = 0; i < len; i++)
642 err_puts(m, ascii85_encode(addr[i], out));
643 }
644 err_puts(m, "\n");
645}
646
647static void err_print_capabilities(struct drm_i915_error_state_buf *m,
648 struct i915_gpu_coredump *error)
649{
650 struct drm_printer p = i915_error_printer(e: m);
651
652 intel_device_info_print(info: &error->device_info, runtime: &error->runtime_info, p: &p);
653 intel_driver_caps_print(caps: &error->driver_caps, p: &p);
654}
655
656static void err_print_params(struct drm_i915_error_state_buf *m,
657 const struct i915_params *params)
658{
659 struct drm_printer p = i915_error_printer(e: m);
660
661 i915_params_dump(params, p: &p);
662}
663
664static void err_print_pciid(struct drm_i915_error_state_buf *m,
665 struct drm_i915_private *i915)
666{
667 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
668
669 err_printf(m, "PCI ID: 0x%04x\n", pdev->device);
670 err_printf(m, "PCI Revision: 0x%02x\n", pdev->revision);
671 err_printf(m, "PCI Subsystem: %04x:%04x\n",
672 pdev->subsystem_vendor,
673 pdev->subsystem_device);
674}
675
676static void err_print_guc_ctb(struct drm_i915_error_state_buf *m,
677 const char *name,
678 const struct intel_ctb_coredump *ctb)
679{
680 if (!ctb->size)
681 return;
682
683 err_printf(m, "GuC %s CTB: raw: 0x%08X, 0x%08X/%08X, cached: 0x%08X/%08X, desc = 0x%08X, buf = 0x%08X x 0x%08X\n",
684 name, ctb->raw_status, ctb->raw_head, ctb->raw_tail,
685 ctb->head, ctb->tail, ctb->desc_offset, ctb->cmds_offset, ctb->size);
686}
687
688/* This list includes registers that are useful in debugging GuC hangs. */
689const struct {
690 u32 start;
691 u32 count;
692} guc_hw_reg_state[] = {
693 { 0xc0b0, 2 },
694 { 0xc000, 65 },
695 { 0xc140, 1 },
696 { 0xc180, 16 },
697 { 0xc1dc, 10 },
698 { 0xc300, 79 },
699 { 0xc4b4, 47 },
700 { 0xc574, 1 },
701 { 0xc57c, 1 },
702 { 0xc584, 11 },
703 { 0xc5c0, 8 },
704 { 0xc5e4, 1 },
705 { 0xc5ec, 103 },
706 { 0xc7c0, 1 },
707 { 0xc0b0, 2 }
708};
709
710static u32 print_range_line(struct drm_i915_error_state_buf *m, u32 start, u32 *dump, u32 count)
711{
712 if (count >= 8) {
713 err_printf(m, "[0x%04x] 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
714 start, dump[0], dump[1], dump[2], dump[3],
715 dump[4], dump[5], dump[6], dump[7]);
716 return 8;
717 } else if (count >= 4) {
718 err_printf(m, "[0x%04x] 0x%08x 0x%08x 0x%08x 0x%08x\n",
719 start, dump[0], dump[1], dump[2], dump[3]);
720 return 4;
721 } else if (count >= 2) {
722 err_printf(m, "[0x%04x] 0x%08x 0x%08x\n", start, dump[0], dump[1]);
723 return 2;
724 }
725
726 err_printf(m, "[0x%04x] 0x%08x\n", start, dump[0]);
727 return 1;
728}
729
730static void err_print_guc_hw_state(struct drm_i915_error_state_buf *m, u32 *hw_state)
731{
732 u32 total = 0;
733 int i;
734
735 if (!hw_state)
736 return;
737
738 err_printf(m, "GuC Register State:\n");
739
740 for (i = 0; i < ARRAY_SIZE(guc_hw_reg_state); i++) {
741 u32 entry = 0;
742
743 while (entry < guc_hw_reg_state[i].count) {
744 u32 start = guc_hw_reg_state[i].start + entry * sizeof(u32);
745 u32 count = guc_hw_reg_state[i].count - entry;
746 u32 *values = hw_state + total + entry;
747
748 entry += print_range_line(m, start, dump: values, count);
749 }
750
751 GEM_BUG_ON(entry != guc_hw_reg_state[i].count);
752 total += entry;
753 }
754}
755
756static void err_print_uc(struct drm_i915_error_state_buf *m,
757 const struct intel_uc_coredump *error_uc)
758{
759 struct drm_printer p = i915_error_printer(e: m);
760
761 intel_uc_fw_dump(uc_fw: &error_uc->guc_fw, p: &p);
762 intel_uc_fw_dump(uc_fw: &error_uc->huc_fw, p: &p);
763 err_printf(m, "GuC timestamp: 0x%08x\n", error_uc->guc.timestamp);
764 err_print_guc_hw_state(m, hw_state: error_uc->guc.hw_state);
765 intel_gpu_error_print_vma(m, NULL, vma: error_uc->guc.vma_log);
766 err_printf(m, "GuC CTB fence: %d\n", error_uc->guc.last_fence);
767 err_print_guc_ctb(m, name: "Send", ctb: error_uc->guc.ctb + 0);
768 err_print_guc_ctb(m, name: "Recv", ctb: error_uc->guc.ctb + 1);
769 intel_gpu_error_print_vma(m, NULL, vma: error_uc->guc.vma_ctb);
770}
771
772static void err_free_sgl(struct scatterlist *sgl)
773{
774 while (sgl) {
775 struct scatterlist *sg;
776
777 for (sg = sgl; !sg_is_chain(sg); sg++) {
778 kfree(objp: sg_virt(sg));
779 if (sg_is_last(sg))
780 break;
781 }
782
783 sg = sg_is_last(sg) ? NULL : sg_chain_ptr(sg);
784 free_page((unsigned long)sgl);
785 sgl = sg;
786 }
787}
788
789static void err_print_gt_info(struct drm_i915_error_state_buf *m,
790 struct intel_gt_coredump *gt)
791{
792 struct drm_printer p = i915_error_printer(e: m);
793
794 intel_gt_info_print(info: &gt->info, p: &p);
795 intel_sseu_print_topology(i915: gt->_gt->i915, sseu: &gt->info.sseu, p: &p);
796}
797
798static void err_print_gt_global_nonguc(struct drm_i915_error_state_buf *m,
799 struct intel_gt_coredump *gt)
800{
801 int i;
802
803 err_printf(m, "GT awake: %s\n", str_yes_no(gt->awake));
804 err_printf(m, "CS timestamp frequency: %u Hz, %d ns\n",
805 gt->clock_frequency, gt->clock_period_ns);
806 err_printf(m, "EIR: 0x%08x\n", gt->eir);
807 err_printf(m, "PGTBL_ER: 0x%08x\n", gt->pgtbl_er);
808
809 for (i = 0; i < gt->ngtier; i++)
810 err_printf(m, "GTIER[%d]: 0x%08x\n", i, gt->gtier[i]);
811}
812
813static void err_print_gt_global(struct drm_i915_error_state_buf *m,
814 struct intel_gt_coredump *gt)
815{
816 err_printf(m, "FORCEWAKE: 0x%08x\n", gt->forcewake);
817
818 if (IS_GRAPHICS_VER(m->i915, 6, 11)) {
819 err_printf(m, "ERROR: 0x%08x\n", gt->error);
820 err_printf(m, "DONE_REG: 0x%08x\n", gt->done_reg);
821 }
822
823 if (GRAPHICS_VER(m->i915) >= 8)
824 err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n",
825 gt->fault_data1, gt->fault_data0);
826
827 if (GRAPHICS_VER(m->i915) == 7)
828 err_printf(m, "ERR_INT: 0x%08x\n", gt->err_int);
829
830 if (IS_GRAPHICS_VER(m->i915, 8, 11))
831 err_printf(m, "GTT_CACHE_EN: 0x%08x\n", gt->gtt_cache);
832
833 if (GRAPHICS_VER(m->i915) == 12)
834 err_printf(m, "AUX_ERR_DBG: 0x%08x\n", gt->aux_err);
835
836 if (GRAPHICS_VER(m->i915) >= 12) {
837 int i;
838
839 for (i = 0; i < I915_MAX_SFC; i++) {
840 /*
841 * SFC_DONE resides in the VD forcewake domain, so it
842 * only exists if the corresponding VCS engine is
843 * present.
844 */
845 if ((gt->_gt->info.sfc_mask & BIT(i)) == 0 ||
846 !HAS_ENGINE(gt->_gt, _VCS(i * 2)))
847 continue;
848
849 err_printf(m, " SFC_DONE[%d]: 0x%08x\n", i,
850 gt->sfc_done[i]);
851 }
852
853 err_printf(m, " GAM_DONE: 0x%08x\n", gt->gam_done);
854 }
855}
856
857static void err_print_gt_fences(struct drm_i915_error_state_buf *m,
858 struct intel_gt_coredump *gt)
859{
860 int i;
861
862 for (i = 0; i < gt->nfence; i++)
863 err_printf(m, " fence[%d] = %08llx\n", i, gt->fence[i]);
864}
865
866static void err_print_gt_engines(struct drm_i915_error_state_buf *m,
867 struct intel_gt_coredump *gt)
868{
869 const struct intel_engine_coredump *ee;
870
871 for (ee = gt->engine; ee; ee = ee->next) {
872 const struct i915_vma_coredump *vma;
873
874 if (gt->uc && gt->uc->guc.is_guc_capture) {
875 if (ee->guc_capture_node)
876 intel_guc_capture_print_engine_node(m, ee);
877 else
878 err_printf(m, " Missing GuC capture node for %s\n",
879 ee->engine->name);
880 } else {
881 error_print_engine(m, ee);
882 }
883
884 err_printf(m, " hung: %u\n", ee->hung);
885 err_printf(m, " engine reset count: %u\n", ee->reset_count);
886 error_print_context(m, header: " Active context: ", ctx: &ee->context);
887
888 for (vma = ee->vma; vma; vma = vma->next)
889 intel_gpu_error_print_vma(m, engine: ee->engine, vma);
890 }
891
892}
893
894static void __err_print_to_sgl(struct drm_i915_error_state_buf *m,
895 struct i915_gpu_coredump *error)
896{
897 struct drm_printer p = i915_error_printer(e: m);
898 const struct intel_engine_coredump *ee;
899 struct timespec64 ts;
900
901 if (*error->error_msg)
902 err_printf(m, "%s\n", error->error_msg);
903 err_printf(m, "Kernel: %s %s\n",
904 init_utsname()->release,
905 init_utsname()->machine);
906 ts = ktime_to_timespec64(error->time);
907 err_printf(m, "Time: %lld s %ld us\n",
908 (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC);
909 ts = ktime_to_timespec64(error->boottime);
910 err_printf(m, "Boottime: %lld s %ld us\n",
911 (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC);
912 ts = ktime_to_timespec64(error->uptime);
913 err_printf(m, "Uptime: %lld s %ld us\n",
914 (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC);
915 err_printf(m, "Capture: %lu jiffies; %d ms ago\n",
916 error->capture, jiffies_to_msecs(jiffies - error->capture));
917
918 for (ee = error->gt ? error->gt->engine : NULL; ee; ee = ee->next)
919 err_printf(m, "Active process (on ring %s): %s [%d]\n",
920 ee->engine->name,
921 ee->context.comm,
922 ee->context.pid);
923
924 err_printf(m, "Reset count: %u\n", error->reset_count);
925 err_printf(m, "Suspend count: %u\n", error->suspend_count);
926 err_printf(m, "Platform: %s\n", intel_platform_name(error->device_info.platform));
927 err_printf(m, "Subplatform: 0x%x\n",
928 intel_subplatform(&error->runtime_info,
929 error->device_info.platform));
930 err_print_pciid(m, i915: m->i915);
931
932 err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
933
934 err_printf(m, "RPM wakelock: %s\n", str_yes_no(error->wakelock));
935 err_printf(m, "PM suspended: %s\n", str_yes_no(error->suspended));
936
937 if (error->gt) {
938 bool print_guc_capture = false;
939
940 if (error->gt->uc && error->gt->uc->guc.is_guc_capture)
941 print_guc_capture = true;
942
943 err_print_gt_global_nonguc(m, gt: error->gt);
944 err_print_gt_fences(m, gt: error->gt);
945
946 /*
947 * GuC dumped global, eng-class and eng-instance registers together
948 * as part of engine state dump so we print in err_print_gt_engines
949 */
950 if (!print_guc_capture)
951 err_print_gt_global(m, gt: error->gt);
952
953 err_print_gt_engines(m, gt: error->gt);
954
955 if (error->gt->uc)
956 err_print_uc(m, error_uc: error->gt->uc);
957
958 err_print_gt_info(m, gt: error->gt);
959 }
960
961 err_print_capabilities(m, error);
962 err_print_params(m, params: &error->params);
963
964 intel_display_snapshot_print(snapshot: error->display_snapshot, p: &p);
965}
966
967static int err_print_to_sgl(struct i915_gpu_coredump *error)
968{
969 struct drm_i915_error_state_buf m;
970
971 if (IS_ERR(ptr: error))
972 return PTR_ERR(ptr: error);
973
974 if (READ_ONCE(error->sgl))
975 return 0;
976
977 memset(s: &m, c: 0, n: sizeof(m));
978 m.i915 = error->i915;
979
980 __err_print_to_sgl(m: &m, error);
981
982 if (m.buf) {
983 __sg_set_buf(sg: m.cur++, addr: m.buf, len: m.bytes, it: m.iter);
984 m.bytes = 0;
985 m.buf = NULL;
986 }
987 if (m.cur) {
988 GEM_BUG_ON(m.end < m.cur);
989 sg_mark_end(sg: m.cur - 1);
990 }
991 GEM_BUG_ON(m.sgl && !m.cur);
992
993 if (m.err) {
994 err_free_sgl(sgl: m.sgl);
995 return m.err;
996 }
997
998 if (cmpxchg(&error->sgl, NULL, m.sgl))
999 err_free_sgl(sgl: m.sgl);
1000
1001 return 0;
1002}
1003
1004ssize_t i915_gpu_coredump_copy_to_buffer(struct i915_gpu_coredump *error,
1005 char *buf, loff_t off, size_t rem)
1006{
1007 struct scatterlist *sg;
1008 size_t count;
1009 loff_t pos;
1010 int err;
1011
1012 if (!error || !rem)
1013 return 0;
1014
1015 err = err_print_to_sgl(error);
1016 if (err)
1017 return err;
1018
1019 sg = READ_ONCE(error->fit);
1020 if (!sg || off < sg->dma_address)
1021 sg = error->sgl;
1022 if (!sg)
1023 return 0;
1024
1025 pos = sg->dma_address;
1026 count = 0;
1027 do {
1028 size_t len, start;
1029
1030 if (sg_is_chain(sg)) {
1031 sg = sg_chain_ptr(sg);
1032 GEM_BUG_ON(sg_is_chain(sg));
1033 }
1034
1035 len = sg->length;
1036 if (pos + len <= off) {
1037 pos += len;
1038 continue;
1039 }
1040
1041 start = sg->offset;
1042 if (pos < off) {
1043 GEM_BUG_ON(off - pos > len);
1044 len -= off - pos;
1045 start += off - pos;
1046 pos = off;
1047 }
1048
1049 len = min(len, rem);
1050 GEM_BUG_ON(!len || len > sg->length);
1051
1052 memcpy(to: buf, page_address(sg_page(sg)) + start, len);
1053
1054 count += len;
1055 pos += len;
1056
1057 buf += len;
1058 rem -= len;
1059 if (!rem) {
1060 WRITE_ONCE(error->fit, sg);
1061 break;
1062 }
1063 } while (!sg_is_last(sg: sg++));
1064
1065 return count;
1066}
1067
1068static void i915_vma_coredump_free(struct i915_vma_coredump *vma)
1069{
1070 while (vma) {
1071 struct i915_vma_coredump *next = vma->next;
1072 struct page *page, *n;
1073
1074 list_for_each_entry_safe(page, n, &vma->page_list, lru) {
1075 list_del_init(entry: &page->lru);
1076 __free_page(page);
1077 }
1078
1079 kfree(objp: vma);
1080 vma = next;
1081 }
1082}
1083
1084static void cleanup_params(struct i915_gpu_coredump *error)
1085{
1086 i915_params_free(params: &error->params);
1087}
1088
1089static void cleanup_uc(struct intel_uc_coredump *uc)
1090{
1091 kfree(objp: uc->guc_fw.file_selected.path);
1092 kfree(objp: uc->huc_fw.file_selected.path);
1093 kfree(objp: uc->guc_fw.file_wanted.path);
1094 kfree(objp: uc->huc_fw.file_wanted.path);
1095 i915_vma_coredump_free(vma: uc->guc.vma_log);
1096 i915_vma_coredump_free(vma: uc->guc.vma_ctb);
1097 kfree(objp: uc->guc.hw_state);
1098
1099 kfree(objp: uc);
1100}
1101
1102static void cleanup_gt(struct intel_gt_coredump *gt)
1103{
1104 while (gt->engine) {
1105 struct intel_engine_coredump *ee = gt->engine;
1106
1107 gt->engine = ee->next;
1108
1109 i915_vma_coredump_free(vma: ee->vma);
1110 intel_guc_capture_free_node(ee);
1111 kfree(objp: ee);
1112 }
1113
1114 if (gt->uc)
1115 cleanup_uc(uc: gt->uc);
1116
1117 kfree(objp: gt);
1118}
1119
1120void __i915_gpu_coredump_free(struct kref *error_ref)
1121{
1122 struct i915_gpu_coredump *error =
1123 container_of(error_ref, typeof(*error), ref);
1124
1125 while (error->gt) {
1126 struct intel_gt_coredump *gt = error->gt;
1127
1128 error->gt = gt->next;
1129 cleanup_gt(gt);
1130 }
1131
1132 intel_display_snapshot_free(snapshot: error->display_snapshot);
1133
1134 cleanup_params(error);
1135
1136 err_free_sgl(sgl: error->sgl);
1137 kfree(objp: error);
1138}
1139
1140static struct i915_vma_coredump *
1141i915_vma_coredump_create(const struct intel_gt *gt,
1142 const struct i915_vma_resource *vma_res,
1143 struct i915_vma_compress *compress,
1144 const char *name)
1145
1146{
1147 struct i915_ggtt *ggtt = gt->ggtt;
1148 const u64 slot = ggtt->error_capture.start;
1149 struct i915_vma_coredump *dst;
1150 struct sgt_iter iter;
1151 int ret;
1152
1153 might_sleep();
1154
1155 if (!vma_res || !vma_res->bi.pages || !compress)
1156 return NULL;
1157
1158 dst = kmalloc(sizeof(*dst), ALLOW_FAIL);
1159 if (!dst)
1160 return NULL;
1161
1162 if (!compress_start(c: compress)) {
1163 kfree(objp: dst);
1164 return NULL;
1165 }
1166
1167 INIT_LIST_HEAD(list: &dst->page_list);
1168 strscpy(dst->name, name);
1169 dst->next = NULL;
1170
1171 dst->gtt_offset = vma_res->start;
1172 dst->gtt_size = vma_res->node_size;
1173 dst->gtt_page_sizes = vma_res->page_sizes_gtt;
1174 dst->unused = 0;
1175
1176 ret = -EINVAL;
1177 if (drm_mm_node_allocated(node: &ggtt->error_capture)) {
1178 void __iomem *s;
1179 dma_addr_t dma;
1180
1181 for_each_sgt_daddr(dma, iter, vma_res->bi.pages) {
1182 mutex_lock(lock: &ggtt->error_mutex);
1183 if (ggtt->vm.raw_insert_page)
1184 ggtt->vm.raw_insert_page(&ggtt->vm, dma, slot,
1185 i915_gem_get_pat_index(i915: gt->i915,
1186 level: I915_CACHE_NONE),
1187 0);
1188 else
1189 ggtt->vm.insert_page(&ggtt->vm, dma, slot,
1190 i915_gem_get_pat_index(i915: gt->i915,
1191 level: I915_CACHE_NONE),
1192 0);
1193 mb();
1194
1195 s = io_mapping_map_wc(mapping: &ggtt->iomap, offset: slot, PAGE_SIZE);
1196 ret = compress_page(c: compress,
1197 src: (void __force *)s, dst,
1198 wc: true);
1199 io_mapping_unmap(vaddr: s);
1200
1201 mb();
1202 ggtt->vm.clear_range(&ggtt->vm, slot, PAGE_SIZE);
1203 mutex_unlock(lock: &ggtt->error_mutex);
1204 if (ret)
1205 break;
1206 }
1207 } else if (vma_res->bi.lmem) {
1208 struct intel_memory_region *mem = vma_res->mr;
1209 dma_addr_t dma;
1210
1211 for_each_sgt_daddr(dma, iter, vma_res->bi.pages) {
1212 dma_addr_t offset = dma - mem->region.start;
1213 void __iomem *s;
1214
1215 if (offset + PAGE_SIZE > resource_size(res: &mem->io)) {
1216 ret = -EINVAL;
1217 break;
1218 }
1219
1220 s = io_mapping_map_wc(mapping: &mem->iomap, offset, PAGE_SIZE);
1221 ret = compress_page(c: compress,
1222 src: (void __force *)s, dst,
1223 wc: true);
1224 io_mapping_unmap(vaddr: s);
1225 if (ret)
1226 break;
1227 }
1228 } else {
1229 struct page *page;
1230
1231 for_each_sgt_page(page, iter, vma_res->bi.pages) {
1232 void *s;
1233
1234 drm_clflush_pages(pages: &page, num_pages: 1);
1235
1236 s = kmap_local_page(page);
1237 ret = compress_page(c: compress, src: s, dst, wc: false);
1238 kunmap_local(s);
1239
1240 drm_clflush_pages(pages: &page, num_pages: 1);
1241
1242 if (ret)
1243 break;
1244 }
1245 }
1246
1247 if (ret || compress_flush(c: compress, dst)) {
1248 struct page *page, *n;
1249
1250 list_for_each_entry_safe_reverse(page, n, &dst->page_list, lru) {
1251 list_del_init(entry: &page->lru);
1252 pool_free(fbatch: &compress->pool, page_address(page));
1253 }
1254
1255 kfree(objp: dst);
1256 dst = NULL;
1257 }
1258 compress_finish(c: compress);
1259
1260 return dst;
1261}
1262
1263static void gt_record_fences(struct intel_gt_coredump *gt)
1264{
1265 struct i915_ggtt *ggtt = gt->_gt->ggtt;
1266 struct intel_uncore *uncore = gt->_gt->uncore;
1267 int i;
1268
1269 if (GRAPHICS_VER(uncore->i915) >= 6) {
1270 for (i = 0; i < ggtt->num_fences; i++)
1271 gt->fence[i] =
1272 intel_uncore_read64(uncore,
1273 FENCE_REG_GEN6_LO(i));
1274 } else if (GRAPHICS_VER(uncore->i915) >= 4) {
1275 for (i = 0; i < ggtt->num_fences; i++)
1276 gt->fence[i] =
1277 intel_uncore_read64(uncore,
1278 FENCE_REG_965_LO(i));
1279 } else {
1280 for (i = 0; i < ggtt->num_fences; i++)
1281 gt->fence[i] =
1282 intel_uncore_read(uncore, FENCE_REG(i));
1283 }
1284 gt->nfence = i;
1285}
1286
1287static void engine_record_registers(struct intel_engine_coredump *ee)
1288{
1289 const struct intel_engine_cs *engine = ee->engine;
1290 struct drm_i915_private *i915 = engine->i915;
1291
1292 if (GRAPHICS_VER(i915) >= 6) {
1293 ee->rc_psmi = ENGINE_READ(engine, RING_PSMI_CTL);
1294
1295 /*
1296 * For the media GT, this ring fault register is not replicated,
1297 * so don't do multicast/replicated register read/write
1298 * operation on it.
1299 */
1300 if (MEDIA_VER(i915) >= 13 && engine->gt->type == GT_MEDIA)
1301 ee->fault_reg = intel_uncore_read(uncore: engine->uncore,
1302 XELPMP_RING_FAULT_REG);
1303 else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
1304 ee->fault_reg = intel_gt_mcr_read_any(gt: engine->gt,
1305 XEHP_RING_FAULT_REG);
1306 else if (GRAPHICS_VER(i915) >= 12)
1307 ee->fault_reg = intel_uncore_read(uncore: engine->uncore,
1308 GEN12_RING_FAULT_REG);
1309 else if (GRAPHICS_VER(i915) >= 8)
1310 ee->fault_reg = intel_uncore_read(uncore: engine->uncore,
1311 GEN8_RING_FAULT_REG);
1312 else
1313 ee->fault_reg = GEN6_RING_FAULT_REG_READ(engine);
1314 }
1315
1316 if (GRAPHICS_VER(i915) >= 4) {
1317 ee->esr = ENGINE_READ(engine, RING_ESR);
1318 ee->faddr = ENGINE_READ(engine, RING_DMA_FADD);
1319 ee->ipeir = ENGINE_READ(engine, RING_IPEIR);
1320 ee->ipehr = ENGINE_READ(engine, RING_IPEHR);
1321 ee->instps = ENGINE_READ(engine, RING_INSTPS);
1322 ee->bbaddr = ENGINE_READ(engine, RING_BBADDR);
1323 ee->ccid = ENGINE_READ(engine, CCID);
1324 if (GRAPHICS_VER(i915) >= 8) {
1325 ee->faddr |= (u64)ENGINE_READ(engine, RING_DMA_FADD_UDW) << 32;
1326 ee->bbaddr |= (u64)ENGINE_READ(engine, RING_BBADDR_UDW) << 32;
1327 }
1328 ee->bbstate = ENGINE_READ(engine, RING_BBSTATE);
1329 } else {
1330 ee->faddr = ENGINE_READ(engine, DMA_FADD_I8XX);
1331 ee->ipeir = ENGINE_READ(engine, IPEIR);
1332 ee->ipehr = ENGINE_READ(engine, IPEHR);
1333 }
1334
1335 if (GRAPHICS_VER(i915) >= 11) {
1336 ee->cmd_cctl = ENGINE_READ(engine, RING_CMD_CCTL);
1337 ee->cscmdop = ENGINE_READ(engine, RING_CSCMDOP);
1338 ee->ctx_sr_ctl = ENGINE_READ(engine, RING_CTX_SR_CTL);
1339 ee->dma_faddr_hi = ENGINE_READ(engine, RING_DMA_FADD_UDW);
1340 ee->dma_faddr_lo = ENGINE_READ(engine, RING_DMA_FADD);
1341 ee->nopid = ENGINE_READ(engine, RING_NOPID);
1342 ee->excc = ENGINE_READ(engine, RING_EXCC);
1343 }
1344
1345 intel_engine_get_instdone(engine, instdone: &ee->instdone);
1346
1347 ee->instpm = ENGINE_READ(engine, RING_INSTPM);
1348 ee->acthd = intel_engine_get_active_head(engine);
1349 ee->start = ENGINE_READ(engine, RING_START);
1350 ee->head = ENGINE_READ(engine, RING_HEAD);
1351 ee->tail = ENGINE_READ(engine, RING_TAIL);
1352 ee->ctl = ENGINE_READ(engine, RING_CTL);
1353 if (GRAPHICS_VER(i915) > 2)
1354 ee->mode = ENGINE_READ(engine, RING_MI_MODE);
1355
1356 if (!HWS_NEEDS_PHYSICAL(i915)) {
1357 i915_reg_t mmio;
1358
1359 if (GRAPHICS_VER(i915) == 7) {
1360 switch (engine->id) {
1361 default:
1362 MISSING_CASE(engine->id);
1363 fallthrough;
1364 case RCS0:
1365 mmio = RENDER_HWS_PGA_GEN7;
1366 break;
1367 case BCS0:
1368 mmio = BLT_HWS_PGA_GEN7;
1369 break;
1370 case VCS0:
1371 mmio = BSD_HWS_PGA_GEN7;
1372 break;
1373 case VECS0:
1374 mmio = VEBOX_HWS_PGA_GEN7;
1375 break;
1376 }
1377 } else if (GRAPHICS_VER(engine->i915) == 6) {
1378 mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
1379 } else {
1380 /* XXX: gen8 returns to sanity */
1381 mmio = RING_HWS_PGA(engine->mmio_base);
1382 }
1383
1384 ee->hws = intel_uncore_read(uncore: engine->uncore, reg: mmio);
1385 }
1386
1387 ee->reset_count = i915_reset_engine_count(error: &i915->gpu_error, engine);
1388
1389 if (HAS_PPGTT(i915)) {
1390 int i;
1391
1392 ee->vm_info.gfx_mode = ENGINE_READ(engine, RING_MODE_GEN7);
1393
1394 if (GRAPHICS_VER(i915) == 6) {
1395 ee->vm_info.pp_dir_base =
1396 ENGINE_READ(engine, RING_PP_DIR_BASE_READ);
1397 } else if (GRAPHICS_VER(i915) == 7) {
1398 ee->vm_info.pp_dir_base =
1399 ENGINE_READ(engine, RING_PP_DIR_BASE);
1400 } else if (GRAPHICS_VER(i915) >= 8) {
1401 u32 base = engine->mmio_base;
1402
1403 for (i = 0; i < 4; i++) {
1404 ee->vm_info.pdp[i] =
1405 intel_uncore_read(uncore: engine->uncore,
1406 GEN8_RING_PDP_UDW(base, i));
1407 ee->vm_info.pdp[i] <<= 32;
1408 ee->vm_info.pdp[i] |=
1409 intel_uncore_read(uncore: engine->uncore,
1410 GEN8_RING_PDP_LDW(base, i));
1411 }
1412 }
1413 }
1414}
1415
1416static void record_request(const struct i915_request *request,
1417 struct i915_request_coredump *erq)
1418{
1419 erq->flags = request->fence.flags;
1420 erq->context = request->fence.context;
1421 erq->seqno = request->fence.seqno;
1422 erq->sched_attr = request->sched.attr;
1423 erq->head = request->head;
1424 erq->tail = request->tail;
1425
1426 erq->pid = 0;
1427 rcu_read_lock();
1428 if (!intel_context_is_closed(ce: request->context)) {
1429 const struct i915_gem_context *ctx;
1430
1431 ctx = rcu_dereference(request->context->gem_context);
1432 if (ctx)
1433 erq->pid = pid_nr(pid: ctx->pid);
1434 }
1435 rcu_read_unlock();
1436}
1437
1438static void engine_record_execlists(struct intel_engine_coredump *ee)
1439{
1440 const struct intel_engine_execlists * const el = &ee->engine->execlists;
1441 struct i915_request * const *port = el->active;
1442 unsigned int n = 0;
1443
1444 while (*port)
1445 record_request(request: *port++, erq: &ee->execlist[n++]);
1446
1447 ee->num_ports = n;
1448}
1449
1450static bool record_context(struct i915_gem_context_coredump *e,
1451 struct intel_context *ce)
1452{
1453 struct i915_gem_context *ctx;
1454 struct task_struct *task;
1455 bool simulated;
1456
1457 rcu_read_lock();
1458 ctx = rcu_dereference(ce->gem_context);
1459 if (ctx && !kref_get_unless_zero(kref: &ctx->ref))
1460 ctx = NULL;
1461 rcu_read_unlock();
1462 if (!ctx)
1463 return true;
1464
1465 rcu_read_lock();
1466 task = pid_task(pid: ctx->pid, PIDTYPE_PID);
1467 if (task) {
1468 strscpy(e->comm, task->comm);
1469 e->pid = task->pid;
1470 }
1471 rcu_read_unlock();
1472
1473 e->sched_attr = ctx->sched;
1474 e->guilty = atomic_read(v: &ctx->guilty_count);
1475 e->active = atomic_read(v: &ctx->active_count);
1476 e->hwsp_seqno = (ce->timeline && ce->timeline->hwsp_seqno) ?
1477 *ce->timeline->hwsp_seqno : ~0U;
1478
1479 e->total_runtime = intel_context_get_total_runtime_ns(ce);
1480 e->avg_runtime = intel_context_get_avg_runtime_ns(ce);
1481
1482 simulated = i915_gem_context_no_error_capture(ctx);
1483
1484 i915_gem_context_put(ctx);
1485 return simulated;
1486}
1487
1488struct intel_engine_capture_vma {
1489 struct intel_engine_capture_vma *next;
1490 struct i915_vma_resource *vma_res;
1491 char name[16];
1492 bool lockdep_cookie;
1493};
1494
1495static struct intel_engine_capture_vma *
1496capture_vma_snapshot(struct intel_engine_capture_vma *next,
1497 struct i915_vma_resource *vma_res,
1498 gfp_t gfp, const char *name)
1499{
1500 struct intel_engine_capture_vma *c;
1501
1502 if (!vma_res)
1503 return next;
1504
1505 c = kmalloc(sizeof(*c), gfp);
1506 if (!c)
1507 return next;
1508
1509 if (!i915_vma_resource_hold(vma_res, lockdep_cookie: &c->lockdep_cookie)) {
1510 kfree(objp: c);
1511 return next;
1512 }
1513
1514 strscpy(c->name, name);
1515 c->vma_res = i915_vma_resource_get(vma_res);
1516
1517 c->next = next;
1518 return c;
1519}
1520
1521static struct intel_engine_capture_vma *
1522capture_vma(struct intel_engine_capture_vma *next,
1523 struct i915_vma *vma,
1524 const char *name,
1525 gfp_t gfp)
1526{
1527 if (!vma)
1528 return next;
1529
1530 /*
1531 * If the vma isn't pinned, then the vma should be snapshotted
1532 * to a struct i915_vma_snapshot at command submission time.
1533 * Not here.
1534 */
1535 if (GEM_WARN_ON(!i915_vma_is_pinned(vma)))
1536 return next;
1537
1538 next = capture_vma_snapshot(next, vma_res: vma->resource, gfp, name);
1539
1540 return next;
1541}
1542
1543static struct intel_engine_capture_vma *
1544capture_user(struct intel_engine_capture_vma *capture,
1545 const struct i915_request *rq,
1546 gfp_t gfp)
1547{
1548 struct i915_capture_list *c;
1549
1550 for (c = rq->capture_list; c; c = c->next)
1551 capture = capture_vma_snapshot(next: capture, vma_res: c->vma_res, gfp,
1552 name: "user");
1553
1554 return capture;
1555}
1556
1557static void add_vma(struct intel_engine_coredump *ee,
1558 struct i915_vma_coredump *vma)
1559{
1560 if (vma) {
1561 vma->next = ee->vma;
1562 ee->vma = vma;
1563 }
1564}
1565
1566static struct i915_vma_coredump *
1567create_vma_coredump(const struct intel_gt *gt, struct i915_vma *vma,
1568 const char *name, struct i915_vma_compress *compress)
1569{
1570 struct i915_vma_coredump *ret = NULL;
1571 struct i915_vma_resource *vma_res;
1572 bool lockdep_cookie;
1573
1574 if (!vma)
1575 return NULL;
1576
1577 vma_res = vma->resource;
1578
1579 if (i915_vma_resource_hold(vma_res, lockdep_cookie: &lockdep_cookie)) {
1580 ret = i915_vma_coredump_create(gt, vma_res, compress, name);
1581 i915_vma_resource_unhold(vma_res, lockdep_cookie);
1582 }
1583
1584 return ret;
1585}
1586
1587static void add_vma_coredump(struct intel_engine_coredump *ee,
1588 const struct intel_gt *gt,
1589 struct i915_vma *vma,
1590 const char *name,
1591 struct i915_vma_compress *compress)
1592{
1593 add_vma(ee, vma: create_vma_coredump(gt, vma, name, compress));
1594}
1595
1596struct intel_engine_coredump *
1597intel_engine_coredump_alloc(struct intel_engine_cs *engine, gfp_t gfp, u32 dump_flags)
1598{
1599 struct intel_engine_coredump *ee;
1600
1601 ee = kzalloc(sizeof(*ee), gfp);
1602 if (!ee)
1603 return NULL;
1604
1605 ee->engine = engine;
1606
1607 if (!(dump_flags & CORE_DUMP_FLAG_IS_GUC_CAPTURE)) {
1608 engine_record_registers(ee);
1609 engine_record_execlists(ee);
1610 }
1611
1612 return ee;
1613}
1614
1615static struct intel_engine_capture_vma *
1616engine_coredump_add_context(struct intel_engine_coredump *ee,
1617 struct intel_context *ce,
1618 gfp_t gfp)
1619{
1620 struct intel_engine_capture_vma *vma = NULL;
1621
1622 ee->simulated |= record_context(e: &ee->context, ce);
1623 if (ee->simulated)
1624 return NULL;
1625
1626 /*
1627 * We need to copy these to an anonymous buffer
1628 * as the simplest method to avoid being overwritten
1629 * by userspace.
1630 */
1631 vma = capture_vma(next: vma, vma: ce->ring->vma, name: "ring", gfp);
1632 vma = capture_vma(next: vma, vma: ce->state, name: "HW context", gfp);
1633
1634 return vma;
1635}
1636
1637struct intel_engine_capture_vma *
1638intel_engine_coredump_add_request(struct intel_engine_coredump *ee,
1639 struct i915_request *rq,
1640 gfp_t gfp)
1641{
1642 struct intel_engine_capture_vma *vma;
1643
1644 vma = engine_coredump_add_context(ee, ce: rq->context, gfp);
1645 if (!vma)
1646 return NULL;
1647
1648 /*
1649 * We need to copy these to an anonymous buffer
1650 * as the simplest method to avoid being overwritten
1651 * by userspace.
1652 */
1653 vma = capture_vma_snapshot(next: vma, vma_res: rq->batch_res, gfp, name: "batch");
1654 vma = capture_user(capture: vma, rq, gfp);
1655
1656 ee->rq_head = rq->head;
1657 ee->rq_post = rq->postfix;
1658 ee->rq_tail = rq->tail;
1659
1660 return vma;
1661}
1662
1663void
1664intel_engine_coredump_add_vma(struct intel_engine_coredump *ee,
1665 struct intel_engine_capture_vma *capture,
1666 struct i915_vma_compress *compress)
1667{
1668 const struct intel_engine_cs *engine = ee->engine;
1669
1670 while (capture) {
1671 struct intel_engine_capture_vma *this = capture;
1672 struct i915_vma_resource *vma_res = this->vma_res;
1673
1674 add_vma(ee,
1675 vma: i915_vma_coredump_create(gt: engine->gt, vma_res,
1676 compress, name: this->name));
1677
1678 i915_vma_resource_unhold(vma_res, lockdep_cookie: this->lockdep_cookie);
1679 i915_vma_resource_put(vma_res);
1680
1681 capture = this->next;
1682 kfree(objp: this);
1683 }
1684
1685 add_vma_coredump(ee, gt: engine->gt, vma: engine->status_page.vma,
1686 name: "HW Status", compress);
1687
1688 add_vma_coredump(ee, gt: engine->gt, vma: engine->wa_ctx.vma,
1689 name: "WA context", compress);
1690}
1691
1692static struct intel_engine_coredump *
1693capture_engine(struct intel_engine_cs *engine,
1694 struct i915_vma_compress *compress,
1695 u32 dump_flags)
1696{
1697 struct intel_engine_capture_vma *capture = NULL;
1698 struct intel_engine_coredump *ee;
1699 struct intel_context *ce = NULL;
1700 struct i915_request *rq = NULL;
1701
1702 ee = intel_engine_coredump_alloc(engine, ALLOW_FAIL, dump_flags);
1703 if (!ee)
1704 return NULL;
1705
1706 intel_engine_get_hung_entity(engine, ce: &ce, rq: &rq);
1707 if (rq && !i915_request_started(rq)) {
1708 /*
1709 * We want to know also what is the guc_id of the context,
1710 * but if we don't have the context reference, then skip
1711 * printing it.
1712 */
1713 if (ce)
1714 drm_info(&engine->gt->i915->drm,
1715 "Got hung context on %s with active request %lld:%lld [0x%04X] not yet started\n",
1716 engine->name, rq->fence.context, rq->fence.seqno, ce->guc_id.id);
1717 else
1718 drm_info(&engine->gt->i915->drm,
1719 "Got hung context on %s with active request %lld:%lld not yet started\n",
1720 engine->name, rq->fence.context, rq->fence.seqno);
1721 }
1722
1723 if (rq) {
1724 capture = intel_engine_coredump_add_request(ee, rq, ATOMIC_MAYFAIL);
1725 i915_request_put(rq);
1726 } else if (ce) {
1727 capture = engine_coredump_add_context(ee, ce, ATOMIC_MAYFAIL);
1728 }
1729
1730 if (capture) {
1731 intel_engine_coredump_add_vma(ee, capture, compress);
1732
1733 if (dump_flags & CORE_DUMP_FLAG_IS_GUC_CAPTURE)
1734 intel_guc_capture_get_matching_node(gt: engine->gt, ee, ce);
1735 } else {
1736 kfree(objp: ee);
1737 ee = NULL;
1738 }
1739
1740 return ee;
1741}
1742
1743static void
1744gt_record_engines(struct intel_gt_coredump *gt,
1745 intel_engine_mask_t engine_mask,
1746 struct i915_vma_compress *compress,
1747 u32 dump_flags)
1748{
1749 struct intel_engine_cs *engine;
1750 enum intel_engine_id id;
1751
1752 for_each_engine(engine, gt->_gt, id) {
1753 struct intel_engine_coredump *ee;
1754
1755 /* Refill our page pool before entering atomic section */
1756 pool_refill(fbatch: &compress->pool, ALLOW_FAIL);
1757
1758 ee = capture_engine(engine, compress, dump_flags);
1759 if (!ee)
1760 continue;
1761
1762 ee->hung = engine->mask & engine_mask;
1763
1764 gt->simulated |= ee->simulated;
1765 if (ee->simulated) {
1766 if (dump_flags & CORE_DUMP_FLAG_IS_GUC_CAPTURE)
1767 intel_guc_capture_free_node(ee);
1768 kfree(objp: ee);
1769 continue;
1770 }
1771
1772 ee->next = gt->engine;
1773 gt->engine = ee;
1774 }
1775}
1776
1777static void gt_record_guc_ctb(struct intel_ctb_coredump *saved,
1778 const struct intel_guc_ct_buffer *ctb,
1779 const void *blob_ptr, struct intel_guc *guc)
1780{
1781 if (!ctb || !ctb->desc)
1782 return;
1783
1784 saved->raw_status = ctb->desc->status;
1785 saved->raw_head = ctb->desc->head;
1786 saved->raw_tail = ctb->desc->tail;
1787 saved->head = ctb->head;
1788 saved->tail = ctb->tail;
1789 saved->size = ctb->size;
1790 saved->desc_offset = ((void *)ctb->desc) - blob_ptr;
1791 saved->cmds_offset = ((void *)ctb->cmds) - blob_ptr;
1792}
1793
1794static u32 read_guc_state_reg(struct intel_uncore *uncore, int range, int count)
1795{
1796 GEM_BUG_ON(range >= ARRAY_SIZE(guc_hw_reg_state));
1797 GEM_BUG_ON(count >= guc_hw_reg_state[range].count);
1798
1799 return intel_uncore_read(uncore,
1800 _MMIO(guc_hw_reg_state[range].start + count * sizeof(u32)));
1801}
1802
1803static void gt_record_guc_hw_state(struct intel_uncore *uncore,
1804 struct intel_uc_coredump *error_uc)
1805{
1806 u32 *hw_state;
1807 u32 count = 0;
1808 int i, j;
1809
1810 for (i = 0; i < ARRAY_SIZE(guc_hw_reg_state); i++)
1811 count += guc_hw_reg_state[i].count;
1812
1813 hw_state = kcalloc(count, sizeof(u32), ALLOW_FAIL);
1814 if (!hw_state)
1815 return;
1816
1817 count = 0;
1818 for (i = 0; i < ARRAY_SIZE(guc_hw_reg_state); i++)
1819 for (j = 0; j < guc_hw_reg_state[i].count; j++)
1820 hw_state[count++] = read_guc_state_reg(uncore, range: i, count: j);
1821
1822 error_uc->guc.hw_state = hw_state;
1823}
1824
1825static struct intel_uc_coredump *
1826gt_record_uc(struct intel_gt_coredump *gt,
1827 struct i915_vma_compress *compress)
1828{
1829 const struct intel_uc *uc = &gt->_gt->uc;
1830 struct intel_uc_coredump *error_uc;
1831
1832 error_uc = kzalloc(sizeof(*error_uc), ALLOW_FAIL);
1833 if (!error_uc)
1834 return NULL;
1835
1836 memcpy(to: &error_uc->guc_fw, from: &uc->guc.fw, len: sizeof(uc->guc.fw));
1837 memcpy(to: &error_uc->huc_fw, from: &uc->huc.fw, len: sizeof(uc->huc.fw));
1838
1839 error_uc->guc_fw.file_selected.path = kstrdup(s: uc->guc.fw.file_selected.path, ALLOW_FAIL);
1840 error_uc->huc_fw.file_selected.path = kstrdup(s: uc->huc.fw.file_selected.path, ALLOW_FAIL);
1841 error_uc->guc_fw.file_wanted.path = kstrdup(s: uc->guc.fw.file_wanted.path, ALLOW_FAIL);
1842 error_uc->huc_fw.file_wanted.path = kstrdup(s: uc->huc.fw.file_wanted.path, ALLOW_FAIL);
1843
1844 /*
1845 * Save the GuC log and include a timestamp reference for converting the
1846 * log times to system times (in conjunction with the error->boottime and
1847 * gt->clock_frequency fields saved elsewhere).
1848 */
1849 error_uc->guc.timestamp = intel_uncore_read(uncore: gt->_gt->uncore, GUCPMTIMESTAMP);
1850 error_uc->guc.vma_log = create_vma_coredump(gt: gt->_gt, vma: uc->guc.log.vma,
1851 name: "GuC log buffer", compress);
1852 error_uc->guc.vma_ctb = create_vma_coredump(gt: gt->_gt, vma: uc->guc.ct.vma,
1853 name: "GuC CT buffer", compress);
1854 error_uc->guc.last_fence = uc->guc.ct.requests.last_fence;
1855 gt_record_guc_ctb(saved: error_uc->guc.ctb + 0, ctb: &uc->guc.ct.ctbs.send,
1856 blob_ptr: uc->guc.ct.ctbs.send.desc, guc: (struct intel_guc *)&uc->guc);
1857 gt_record_guc_ctb(saved: error_uc->guc.ctb + 1, ctb: &uc->guc.ct.ctbs.recv,
1858 blob_ptr: uc->guc.ct.ctbs.send.desc, guc: (struct intel_guc *)&uc->guc);
1859 gt_record_guc_hw_state(uncore: gt->_gt->uncore, error_uc);
1860
1861 return error_uc;
1862}
1863
1864/* Capture all other registers that GuC doesn't capture. */
1865static void gt_record_global_nonguc_regs(struct intel_gt_coredump *gt)
1866{
1867 struct intel_uncore *uncore = gt->_gt->uncore;
1868 struct drm_i915_private *i915 = uncore->i915;
1869 int i;
1870
1871 if (IS_VALLEYVIEW(i915)) {
1872 gt->gtier[0] = intel_uncore_read(uncore, GTIER);
1873 gt->ngtier = 1;
1874 } else if (GRAPHICS_VER(i915) >= 11) {
1875 gt->gtier[0] =
1876 intel_uncore_read(uncore,
1877 GEN11_RENDER_COPY_INTR_ENABLE);
1878 gt->gtier[1] =
1879 intel_uncore_read(uncore, GEN11_VCS_VECS_INTR_ENABLE);
1880 gt->gtier[2] =
1881 intel_uncore_read(uncore, GEN11_GUC_SG_INTR_ENABLE);
1882 gt->gtier[3] =
1883 intel_uncore_read(uncore,
1884 GEN11_GPM_WGBOXPERF_INTR_ENABLE);
1885 gt->gtier[4] =
1886 intel_uncore_read(uncore,
1887 GEN11_CRYPTO_RSVD_INTR_ENABLE);
1888 gt->gtier[5] =
1889 intel_uncore_read(uncore,
1890 GEN11_GUNIT_CSME_INTR_ENABLE);
1891 gt->ngtier = 6;
1892 } else if (GRAPHICS_VER(i915) >= 8) {
1893 for (i = 0; i < 4; i++)
1894 gt->gtier[i] =
1895 intel_uncore_read(uncore, GEN8_GT_IER(i));
1896 gt->ngtier = 4;
1897 } else if (GRAPHICS_VER(i915) >= 5) {
1898 gt->gtier[0] = intel_uncore_read(uncore, GTIER);
1899 gt->ngtier = 1;
1900 } else {
1901 gt->gtier[0] = intel_uncore_read(uncore, GEN2_IER);
1902 gt->ngtier = 1;
1903 }
1904
1905 gt->eir = intel_uncore_read(uncore, EIR);
1906 gt->pgtbl_er = intel_uncore_read(uncore, PGTBL_ER);
1907}
1908
1909/*
1910 * Capture all registers that relate to workload submission.
1911 * NOTE: In GuC submission, when GuC resets an engine, it can dump these for us
1912 */
1913static void gt_record_global_regs(struct intel_gt_coredump *gt)
1914{
1915 struct intel_uncore *uncore = gt->_gt->uncore;
1916 struct drm_i915_private *i915 = uncore->i915;
1917 int i;
1918
1919 /*
1920 * General organization
1921 * 1. Registers specific to a single generation
1922 * 2. Registers which belong to multiple generations
1923 * 3. Feature specific registers.
1924 * 4. Everything else
1925 * Please try to follow the order.
1926 */
1927
1928 /* 1: Registers specific to a single generation */
1929 if (IS_VALLEYVIEW(i915))
1930 gt->forcewake = intel_uncore_read_fw(uncore, FORCEWAKE_VLV);
1931
1932 if (GRAPHICS_VER(i915) == 7)
1933 gt->err_int = intel_uncore_read(uncore, GEN7_ERR_INT);
1934
1935 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
1936 gt->fault_data0 = intel_gt_mcr_read_any(gt: (struct intel_gt *)gt->_gt,
1937 XEHP_FAULT_TLB_DATA0);
1938 gt->fault_data1 = intel_gt_mcr_read_any(gt: (struct intel_gt *)gt->_gt,
1939 XEHP_FAULT_TLB_DATA1);
1940 } else if (GRAPHICS_VER(i915) >= 12) {
1941 gt->fault_data0 = intel_uncore_read(uncore,
1942 GEN12_FAULT_TLB_DATA0);
1943 gt->fault_data1 = intel_uncore_read(uncore,
1944 GEN12_FAULT_TLB_DATA1);
1945 } else if (GRAPHICS_VER(i915) >= 8) {
1946 gt->fault_data0 = intel_uncore_read(uncore,
1947 GEN8_FAULT_TLB_DATA0);
1948 gt->fault_data1 = intel_uncore_read(uncore,
1949 GEN8_FAULT_TLB_DATA1);
1950 }
1951
1952 if (GRAPHICS_VER(i915) == 6) {
1953 gt->forcewake = intel_uncore_read_fw(uncore, FORCEWAKE);
1954 gt->gab_ctl = intel_uncore_read(uncore, GAB_CTL);
1955 gt->gfx_mode = intel_uncore_read(uncore, GFX_MODE);
1956 }
1957
1958 /* 2: Registers which belong to multiple generations */
1959 if (GRAPHICS_VER(i915) >= 7)
1960 gt->forcewake = intel_uncore_read_fw(uncore, FORCEWAKE_MT);
1961
1962 if (GRAPHICS_VER(i915) >= 6) {
1963 if (GRAPHICS_VER(i915) < 12) {
1964 gt->error = intel_uncore_read(uncore, ERROR_GEN6);
1965 gt->done_reg = intel_uncore_read(uncore, DONE_REG);
1966 }
1967 }
1968
1969 /* 3: Feature specific registers */
1970 if (IS_GRAPHICS_VER(i915, 6, 7)) {
1971 gt->gam_ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
1972 gt->gac_eco = intel_uncore_read(uncore, GAC_ECO_BITS);
1973 }
1974
1975 if (IS_GRAPHICS_VER(i915, 8, 11))
1976 gt->gtt_cache = intel_uncore_read(uncore, HSW_GTT_CACHE_EN);
1977
1978 if (GRAPHICS_VER(i915) == 12)
1979 gt->aux_err = intel_uncore_read(uncore, GEN12_AUX_ERR_DBG);
1980
1981 if (GRAPHICS_VER(i915) >= 12) {
1982 for (i = 0; i < I915_MAX_SFC; i++) {
1983 /*
1984 * SFC_DONE resides in the VD forcewake domain, so it
1985 * only exists if the corresponding VCS engine is
1986 * present.
1987 */
1988 if ((gt->_gt->info.sfc_mask & BIT(i)) == 0 ||
1989 !HAS_ENGINE(gt->_gt, _VCS(i * 2)))
1990 continue;
1991
1992 gt->sfc_done[i] =
1993 intel_uncore_read(uncore, GEN12_SFC_DONE(i));
1994 }
1995
1996 gt->gam_done = intel_uncore_read(uncore, GEN12_GAM_DONE);
1997 }
1998}
1999
2000static void gt_record_info(struct intel_gt_coredump *gt)
2001{
2002 memcpy(to: &gt->info, from: &gt->_gt->info, len: sizeof(struct intel_gt_info));
2003 gt->clock_frequency = gt->_gt->clock_frequency;
2004 gt->clock_period_ns = gt->_gt->clock_period_ns;
2005}
2006
2007/*
2008 * Generate a semi-unique error code. The code is not meant to have meaning, The
2009 * code's only purpose is to try to prevent false duplicated bug reports by
2010 * grossly estimating a GPU error state.
2011 *
2012 * TODO Ideally, hashing the batchbuffer would be a very nice way to determine
2013 * the hang if we could strip the GTT offset information from it.
2014 *
2015 * It's only a small step better than a random number in its current form.
2016 */
2017static u32 generate_ecode(const struct intel_engine_coredump *ee)
2018{
2019 /*
2020 * IPEHR would be an ideal way to detect errors, as it's the gross
2021 * measure of "the command that hung." However, has some very common
2022 * synchronization commands which almost always appear in the case
2023 * strictly a client bug. Use instdone to differentiate those some.
2024 */
2025 return ee ? ee->ipehr ^ ee->instdone.instdone : 0;
2026}
2027
2028static const char *error_msg(struct i915_gpu_coredump *error)
2029{
2030 struct intel_engine_coredump *first = NULL;
2031 unsigned int hung_classes = 0;
2032 struct intel_gt_coredump *gt;
2033 int len;
2034
2035 for (gt = error->gt; gt; gt = gt->next) {
2036 struct intel_engine_coredump *cs;
2037
2038 for (cs = gt->engine; cs; cs = cs->next) {
2039 if (cs->hung) {
2040 hung_classes |= BIT(cs->engine->uabi_class);
2041 if (!first)
2042 first = cs;
2043 }
2044 }
2045 }
2046
2047 len = scnprintf(buf: error->error_msg, size: sizeof(error->error_msg),
2048 fmt: "GPU HANG: ecode %d:%x:%08x",
2049 GRAPHICS_VER(error->i915), hung_classes,
2050 generate_ecode(ee: first));
2051 if (first && first->context.pid) {
2052 /* Just show the first executing process, more is confusing */
2053 len += scnprintf(buf: error->error_msg + len,
2054 size: sizeof(error->error_msg) - len,
2055 fmt: ", in %s [%d]",
2056 first->context.comm, first->context.pid);
2057 }
2058
2059 return error->error_msg;
2060}
2061
2062static void capture_gen(struct i915_gpu_coredump *error)
2063{
2064 struct drm_i915_private *i915 = error->i915;
2065
2066 error->wakelock = atomic_read(v: &i915->runtime_pm.wakeref_count);
2067 error->suspended = pm_runtime_suspended(dev: i915->drm.dev);
2068
2069 error->iommu = i915_vtd_active(i915);
2070 error->reset_count = i915_reset_count(error: &i915->gpu_error);
2071 error->suspend_count = i915->suspend_count;
2072
2073 i915_params_copy(dest: &error->params, src: &i915->params);
2074 memcpy(to: &error->device_info,
2075 INTEL_INFO(i915),
2076 len: sizeof(error->device_info));
2077 memcpy(to: &error->runtime_info,
2078 RUNTIME_INFO(i915),
2079 len: sizeof(error->runtime_info));
2080 error->driver_caps = i915->caps;
2081}
2082
2083struct i915_gpu_coredump *
2084i915_gpu_coredump_alloc(struct drm_i915_private *i915, gfp_t gfp)
2085{
2086 struct i915_gpu_coredump *error;
2087
2088 if (!i915->params.error_capture)
2089 return NULL;
2090
2091 error = kzalloc(sizeof(*error), gfp);
2092 if (!error)
2093 return NULL;
2094
2095 kref_init(kref: &error->ref);
2096 error->i915 = i915;
2097
2098 error->time = ktime_get_real();
2099 error->boottime = ktime_get_boottime();
2100 error->uptime = ktime_sub(ktime_get(), to_gt(i915)->last_init_time);
2101 error->capture = jiffies;
2102
2103 capture_gen(error);
2104
2105 return error;
2106}
2107
2108#define DAY_AS_SECONDS(x) (24 * 60 * 60 * (x))
2109
2110struct intel_gt_coredump *
2111intel_gt_coredump_alloc(struct intel_gt *gt, gfp_t gfp, u32 dump_flags)
2112{
2113 struct intel_gt_coredump *gc;
2114
2115 gc = kzalloc(sizeof(*gc), gfp);
2116 if (!gc)
2117 return NULL;
2118
2119 gc->_gt = gt;
2120 gc->awake = intel_gt_pm_is_awake(gt);
2121
2122 gt_record_global_nonguc_regs(gt: gc);
2123
2124 /*
2125 * GuC dumps global, eng-class and eng-instance registers
2126 * (that can change as part of engine state during execution)
2127 * before an engine is reset due to a hung context.
2128 * GuC captures and reports all three groups of registers
2129 * together as a single set before the engine is reset.
2130 * Thus, if GuC triggered the context reset we retrieve
2131 * the register values as part of gt_record_engines.
2132 */
2133 if (!(dump_flags & CORE_DUMP_FLAG_IS_GUC_CAPTURE))
2134 gt_record_global_regs(gt: gc);
2135
2136 gt_record_fences(gt: gc);
2137
2138 return gc;
2139}
2140
2141struct i915_vma_compress *
2142i915_vma_capture_prepare(struct intel_gt_coredump *gt)
2143{
2144 struct i915_vma_compress *compress;
2145
2146 compress = kmalloc(sizeof(*compress), ALLOW_FAIL);
2147 if (!compress)
2148 return NULL;
2149
2150 if (!compress_init(c: compress)) {
2151 kfree(objp: compress);
2152 return NULL;
2153 }
2154
2155 return compress;
2156}
2157
2158void i915_vma_capture_finish(struct intel_gt_coredump *gt,
2159 struct i915_vma_compress *compress)
2160{
2161 if (!compress)
2162 return;
2163
2164 compress_fini(c: compress);
2165 kfree(objp: compress);
2166}
2167
2168static struct i915_gpu_coredump *
2169__i915_gpu_coredump(struct intel_gt *gt, intel_engine_mask_t engine_mask, u32 dump_flags)
2170{
2171 struct drm_i915_private *i915 = gt->i915;
2172 struct intel_display *display = i915->display;
2173 struct i915_gpu_coredump *error;
2174
2175 /* Check if GPU capture has been disabled */
2176 error = READ_ONCE(i915->gpu_error.first_error);
2177 if (IS_ERR(ptr: error))
2178 return error;
2179
2180 error = i915_gpu_coredump_alloc(i915, ALLOW_FAIL);
2181 if (!error)
2182 return ERR_PTR(error: -ENOMEM);
2183
2184 error->gt = intel_gt_coredump_alloc(gt, ALLOW_FAIL, dump_flags);
2185 if (error->gt) {
2186 struct i915_vma_compress *compress;
2187
2188 compress = i915_vma_capture_prepare(gt: error->gt);
2189 if (!compress) {
2190 kfree(objp: error->gt);
2191 kfree(objp: error);
2192 return ERR_PTR(error: -ENOMEM);
2193 }
2194
2195 if (INTEL_INFO(i915)->has_gt_uc) {
2196 error->gt->uc = gt_record_uc(gt: error->gt, compress);
2197 if (error->gt->uc) {
2198 if (dump_flags & CORE_DUMP_FLAG_IS_GUC_CAPTURE)
2199 error->gt->uc->guc.is_guc_capture = true;
2200 else
2201 GEM_BUG_ON(error->gt->uc->guc.is_guc_capture);
2202 }
2203 }
2204
2205 gt_record_info(gt: error->gt);
2206 gt_record_engines(gt: error->gt, engine_mask, compress, dump_flags);
2207
2208
2209 i915_vma_capture_finish(gt: error->gt, compress);
2210
2211 error->simulated |= error->gt->simulated;
2212 }
2213
2214 error->display_snapshot = intel_display_snapshot_capture(display);
2215
2216 return error;
2217}
2218
2219static struct i915_gpu_coredump *
2220i915_gpu_coredump(struct intel_gt *gt, intel_engine_mask_t engine_mask, u32 dump_flags)
2221{
2222 static DEFINE_MUTEX(capture_mutex);
2223 int ret = mutex_lock_interruptible(lock: &capture_mutex);
2224 struct i915_gpu_coredump *dump;
2225
2226 if (ret)
2227 return ERR_PTR(error: ret);
2228
2229 dump = __i915_gpu_coredump(gt, engine_mask, dump_flags);
2230 mutex_unlock(lock: &capture_mutex);
2231
2232 return dump;
2233}
2234
2235void i915_error_state_store(struct i915_gpu_coredump *error)
2236{
2237 struct drm_i915_private *i915;
2238
2239 if (IS_ERR_OR_NULL(ptr: error))
2240 return;
2241
2242 i915 = error->i915;
2243 drm_info(&i915->drm, "%s\n", error_msg(error));
2244
2245 if (error->simulated ||
2246 cmpxchg(&i915->gpu_error.first_error, NULL, error))
2247 return;
2248
2249 i915_gpu_coredump_get(gpu: error);
2250
2251 drm_info(&i915->drm, "GPU error state saved to /sys/class/drm/card%d/error\n",
2252 i915->drm.primary->index);
2253}
2254
2255/**
2256 * i915_capture_error_state - capture an error record for later analysis
2257 * @gt: intel_gt which originated the hang
2258 * @engine_mask: hung engines
2259 * @dump_flags: dump flags
2260 *
2261 * Should be called when an error is detected (either a hang or an error
2262 * interrupt) to capture error state from the time of the error. Fills
2263 * out a structure which becomes available in debugfs for user level tools
2264 * to pick up.
2265 */
2266void i915_capture_error_state(struct intel_gt *gt,
2267 intel_engine_mask_t engine_mask, u32 dump_flags)
2268{
2269 struct i915_gpu_coredump *error;
2270
2271 error = i915_gpu_coredump(gt, engine_mask, dump_flags);
2272 if (IS_ERR(ptr: error)) {
2273 cmpxchg(&gt->i915->gpu_error.first_error, NULL, error);
2274 return;
2275 }
2276
2277 i915_error_state_store(error);
2278 i915_gpu_coredump_put(gpu: error);
2279}
2280
2281static struct i915_gpu_coredump *
2282i915_first_error_state(struct drm_i915_private *i915)
2283{
2284 struct i915_gpu_coredump *error;
2285
2286 spin_lock_irq(lock: &i915->gpu_error.lock);
2287 error = i915->gpu_error.first_error;
2288 if (!IS_ERR_OR_NULL(ptr: error))
2289 i915_gpu_coredump_get(gpu: error);
2290 spin_unlock_irq(lock: &i915->gpu_error.lock);
2291
2292 return error;
2293}
2294
2295void i915_reset_error_state(struct drm_i915_private *i915)
2296{
2297 struct i915_gpu_coredump *error;
2298
2299 spin_lock_irq(lock: &i915->gpu_error.lock);
2300 error = i915->gpu_error.first_error;
2301 if (error != ERR_PTR(error: -ENODEV)) /* if disabled, always disabled */
2302 i915->gpu_error.first_error = NULL;
2303 spin_unlock_irq(lock: &i915->gpu_error.lock);
2304
2305 if (!IS_ERR_OR_NULL(ptr: error))
2306 i915_gpu_coredump_put(gpu: error);
2307}
2308
2309void i915_disable_error_state(struct drm_i915_private *i915, int err)
2310{
2311 spin_lock_irq(lock: &i915->gpu_error.lock);
2312 if (!i915->gpu_error.first_error)
2313 i915->gpu_error.first_error = ERR_PTR(error: err);
2314 spin_unlock_irq(lock: &i915->gpu_error.lock);
2315}
2316
2317#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
2318void intel_klog_error_capture(struct intel_gt *gt,
2319 intel_engine_mask_t engine_mask)
2320{
2321 static int g_count;
2322 struct drm_i915_private *i915 = gt->i915;
2323 struct i915_gpu_coredump *error;
2324 intel_wakeref_t wakeref;
2325 size_t buf_size = PAGE_SIZE * 128;
2326 size_t pos_err;
2327 char *buf, *ptr, *next;
2328 int l_count = g_count++;
2329 int line = 0;
2330
2331 /* Can't allocate memory during a reset */
2332 if (test_bit(I915_RESET_BACKOFF, &gt->reset.flags)) {
2333 drm_err(&gt->i915->drm, "[Capture/%d.%d] Inside GT reset, skipping error capture :(\n",
2334 l_count, line++);
2335 return;
2336 }
2337
2338 error = READ_ONCE(i915->gpu_error.first_error);
2339 if (error) {
2340 drm_err(&i915->drm, "[Capture/%d.%d] Clearing existing error capture first...\n",
2341 l_count, line++);
2342 i915_reset_error_state(i915);
2343 }
2344
2345 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2346 error = i915_gpu_coredump(gt, engine_mask, CORE_DUMP_FLAG_NONE);
2347
2348 if (IS_ERR(error)) {
2349 drm_err(&i915->drm, "[Capture/%d.%d] Failed to capture error capture: %ld!\n",
2350 l_count, line++, PTR_ERR(error));
2351 return;
2352 }
2353
2354 buf = kvmalloc(buf_size, GFP_KERNEL);
2355 if (!buf) {
2356 drm_err(&i915->drm, "[Capture/%d.%d] Failed to allocate buffer for error capture!\n",
2357 l_count, line++);
2358 i915_gpu_coredump_put(error);
2359 return;
2360 }
2361
2362 drm_info(&i915->drm, "[Capture/%d.%d] Dumping i915 error capture for %ps...\n",
2363 l_count, line++, __builtin_return_address(0));
2364
2365 /* Largest string length safe to print via dmesg */
2366# define MAX_CHUNK 800
2367
2368 pos_err = 0;
2369 while (1) {
2370 ssize_t got = i915_gpu_coredump_copy_to_buffer(error, buf, pos_err, buf_size - 1);
2371
2372 if (got <= 0)
2373 break;
2374
2375 buf[got] = 0;
2376 pos_err += got;
2377
2378 ptr = buf;
2379 while (got > 0) {
2380 size_t count;
2381 char tag[2];
2382
2383 next = strnchr(ptr, got, '\n');
2384 if (next) {
2385 count = next - ptr;
2386 *next = 0;
2387 tag[0] = '>';
2388 tag[1] = '<';
2389 } else {
2390 count = got;
2391 tag[0] = '}';
2392 tag[1] = '{';
2393 }
2394
2395 if (count > MAX_CHUNK) {
2396 size_t pos;
2397 char *ptr2 = ptr;
2398
2399 for (pos = MAX_CHUNK; pos < count; pos += MAX_CHUNK) {
2400 char chr = ptr[pos];
2401
2402 ptr[pos] = 0;
2403 drm_info(&i915->drm, "[Capture/%d.%d] }%s{\n",
2404 l_count, line++, ptr2);
2405 ptr[pos] = chr;
2406 ptr2 = ptr + pos;
2407
2408 /*
2409 * If spewing large amounts of data via a serial console,
2410 * this can be a very slow process. So be friendly and try
2411 * not to cause 'softlockup on CPU' problems.
2412 */
2413 cond_resched();
2414 }
2415
2416 if (ptr2 < (ptr + count))
2417 drm_info(&i915->drm, "[Capture/%d.%d] %c%s%c\n",
2418 l_count, line++, tag[0], ptr2, tag[1]);
2419 else if (tag[0] == '>')
2420 drm_info(&i915->drm, "[Capture/%d.%d] ><\n",
2421 l_count, line++);
2422 } else {
2423 drm_info(&i915->drm, "[Capture/%d.%d] %c%s%c\n",
2424 l_count, line++, tag[0], ptr, tag[1]);
2425 }
2426
2427 ptr = next;
2428 got -= count;
2429 if (next) {
2430 ptr++;
2431 got--;
2432 }
2433
2434 /* As above. */
2435 cond_resched();
2436 }
2437
2438 if (got)
2439 drm_info(&i915->drm, "[Capture/%d.%d] Got %zd bytes remaining!\n",
2440 l_count, line++, got);
2441 }
2442
2443 kvfree(buf);
2444
2445 drm_info(&i915->drm, "[Capture/%d.%d] Dumped %zd bytes\n", l_count, line++, pos_err);
2446}
2447#endif
2448
2449static ssize_t gpu_state_read(struct file *file, char __user *ubuf,
2450 size_t count, loff_t *pos)
2451{
2452 struct i915_gpu_coredump *error;
2453 ssize_t ret;
2454 void *buf;
2455
2456 error = file->private_data;
2457 if (!error)
2458 return 0;
2459
2460 /* Bounce buffer required because of kernfs __user API convenience. */
2461 buf = kmalloc(count, GFP_KERNEL);
2462 if (!buf)
2463 return -ENOMEM;
2464
2465 ret = i915_gpu_coredump_copy_to_buffer(error, buf, off: *pos, rem: count);
2466 if (ret <= 0)
2467 goto out;
2468
2469 if (!copy_to_user(to: ubuf, from: buf, n: ret))
2470 *pos += ret;
2471 else
2472 ret = -EFAULT;
2473
2474out:
2475 kfree(objp: buf);
2476 return ret;
2477}
2478
2479static int gpu_state_release(struct inode *inode, struct file *file)
2480{
2481 i915_gpu_coredump_put(gpu: file->private_data);
2482 return 0;
2483}
2484
2485static int i915_gpu_info_open(struct inode *inode, struct file *file)
2486{
2487 struct drm_i915_private *i915 = inode->i_private;
2488 struct i915_gpu_coredump *gpu;
2489 intel_wakeref_t wakeref;
2490
2491 gpu = NULL;
2492 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2493 gpu = i915_gpu_coredump(gt: to_gt(i915), ALL_ENGINES, CORE_DUMP_FLAG_NONE);
2494
2495 if (IS_ERR(ptr: gpu))
2496 return PTR_ERR(ptr: gpu);
2497
2498 file->private_data = gpu;
2499 return 0;
2500}
2501
2502static const struct file_operations i915_gpu_info_fops = {
2503 .owner = THIS_MODULE,
2504 .open = i915_gpu_info_open,
2505 .read = gpu_state_read,
2506 .llseek = default_llseek,
2507 .release = gpu_state_release,
2508};
2509
2510static ssize_t
2511i915_error_state_write(struct file *filp,
2512 const char __user *ubuf,
2513 size_t cnt,
2514 loff_t *ppos)
2515{
2516 struct i915_gpu_coredump *error = filp->private_data;
2517
2518 if (!error)
2519 return 0;
2520
2521 drm_dbg(&error->i915->drm, "Resetting error state\n");
2522 i915_reset_error_state(i915: error->i915);
2523
2524 return cnt;
2525}
2526
2527static int i915_error_state_open(struct inode *inode, struct file *file)
2528{
2529 struct i915_gpu_coredump *error;
2530
2531 error = i915_first_error_state(i915: inode->i_private);
2532 if (IS_ERR(ptr: error))
2533 return PTR_ERR(ptr: error);
2534
2535 file->private_data = error;
2536 return 0;
2537}
2538
2539static const struct file_operations i915_error_state_fops = {
2540 .owner = THIS_MODULE,
2541 .open = i915_error_state_open,
2542 .read = gpu_state_read,
2543 .write = i915_error_state_write,
2544 .llseek = default_llseek,
2545 .release = gpu_state_release,
2546};
2547
2548void i915_gpu_error_debugfs_register(struct drm_i915_private *i915)
2549{
2550 struct dentry *debugfs_root = i915->drm.debugfs_root;
2551
2552 debugfs_create_file("i915_error_state", 0644, debugfs_root, i915,
2553 &i915_error_state_fops);
2554 debugfs_create_file("i915_gpu_info", 0644, debugfs_root, i915,
2555 &i915_gpu_info_fops);
2556}
2557
2558static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
2559 const struct bin_attribute *attr, char *buf,
2560 loff_t off, size_t count)
2561{
2562
2563 struct device *kdev = kobj_to_dev(kobj);
2564 struct drm_i915_private *i915 = kdev_minor_to_i915(kdev);
2565 struct i915_gpu_coredump *gpu;
2566 ssize_t ret = 0;
2567
2568 /*
2569 * FIXME: Concurrent clients triggering resets and reading + clearing
2570 * dumps can cause inconsistent sysfs reads when a user calls in with a
2571 * non-zero offset to complete a prior partial read but the
2572 * gpu_coredump has been cleared or replaced.
2573 */
2574
2575 gpu = i915_first_error_state(i915);
2576 if (IS_ERR(ptr: gpu)) {
2577 ret = PTR_ERR(ptr: gpu);
2578 } else if (gpu) {
2579 ret = i915_gpu_coredump_copy_to_buffer(error: gpu, buf, off, rem: count);
2580 i915_gpu_coredump_put(gpu);
2581 } else {
2582 const char *str = "No error state collected\n";
2583 size_t len = strlen(str);
2584
2585 if (off < len) {
2586 ret = min_t(size_t, count, len - off);
2587 memcpy(to: buf, from: str + off, len: ret);
2588 }
2589 }
2590
2591 return ret;
2592}
2593
2594static ssize_t error_state_write(struct file *file, struct kobject *kobj,
2595 const struct bin_attribute *attr, char *buf,
2596 loff_t off, size_t count)
2597{
2598 struct device *kdev = kobj_to_dev(kobj);
2599 struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
2600
2601 drm_dbg(&dev_priv->drm, "Resetting error state\n");
2602 i915_reset_error_state(i915: dev_priv);
2603
2604 return count;
2605}
2606
2607static const struct bin_attribute error_state_attr = {
2608 .attr.name = "error",
2609 .attr.mode = S_IRUSR | S_IWUSR,
2610 .size = 0,
2611 .read = error_state_read,
2612 .write = error_state_write,
2613};
2614
2615void i915_gpu_error_sysfs_setup(struct drm_i915_private *i915)
2616{
2617 struct device *kdev = i915->drm.primary->kdev;
2618
2619 if (sysfs_create_bin_file(kobj: &kdev->kobj, attr: &error_state_attr))
2620 drm_err(&i915->drm, "error_state sysfs setup failed\n");
2621}
2622
2623void i915_gpu_error_sysfs_teardown(struct drm_i915_private *i915)
2624{
2625 struct device *kdev = i915->drm.primary->kdev;
2626
2627 sysfs_remove_bin_file(kobj: &kdev->kobj, attr: &error_state_attr);
2628}
2629