1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2016-2019 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/ktime.h>
8#include <linux/string_helpers.h>
9#include <linux/time64.h>
10#include <linux/timekeeping.h>
11
12#include "i915_drv.h"
13#include "i915_wait_util.h"
14#include "intel_guc_ct.h"
15#include "intel_guc_print.h"
16
17#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
18enum {
19 CT_DEAD_ALIVE = 0,
20 CT_DEAD_SETUP,
21 CT_DEAD_WRITE,
22 CT_DEAD_DEADLOCK,
23 CT_DEAD_H2G_HAS_ROOM,
24 CT_DEAD_READ,
25 CT_DEAD_PROCESS_FAILED,
26};
27
28static void ct_dead_ct_worker_func(struct work_struct *w);
29
30#define CT_DEAD(ct, reason) \
31 do { \
32 if (!(ct)->dead_ct_reported) { \
33 (ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \
34 queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \
35 } \
36 } while (0)
37#else
38#define CT_DEAD(ct, reason) do { } while (0)
39#endif
40
41static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
42{
43 return container_of(ct, struct intel_guc, ct);
44}
45
46#define CT_ERROR(_ct, _fmt, ...) \
47 guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
48#ifdef CONFIG_DRM_I915_DEBUG_GUC
49#define CT_DEBUG(_ct, _fmt, ...) \
50 guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
51#else
52#define CT_DEBUG(...) do { } while (0)
53#endif
54#define CT_PROBE_ERROR(_ct, _fmt, ...) \
55 guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__)
56
57/**
58 * DOC: CTB Blob
59 *
60 * We allocate single blob to hold both CTB descriptors and buffers:
61 *
62 * +--------+-----------------------------------------------+------+
63 * | offset | contents | size |
64 * +========+===============================================+======+
65 * | 0x0000 | H2G `CTB Descriptor`_ (send) | |
66 * +--------+-----------------------------------------------+ 4K |
67 * | 0x0800 | G2H `CTB Descriptor`_ (recv) | |
68 * +--------+-----------------------------------------------+------+
69 * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K |
70 * | | | |
71 * +--------+-----------------------------------------------+------+
72 * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K |
73 * | + n*4K | | |
74 * +--------+-----------------------------------------------+------+
75 *
76 * Size of each `CT Buffer`_ must be multiple of 4K.
77 * We don't expect too many messages in flight at any time, unless we are
78 * using the GuC submission. In that case each request requires a minimum
79 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
80 * enough space to avoid backpressure on the driver. We increase the size
81 * of the receive buffer (relative to the send) to ensure a G2H response
82 * CTB has a landing spot.
83 */
84#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
85#define CTB_H2G_BUFFER_SIZE (SZ_4K)
86#define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE)
87#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4)
88
89struct ct_request {
90 struct list_head link;
91 u32 fence;
92 u32 status;
93 u32 response_len;
94 u32 *response_buf;
95};
96
97struct ct_incoming_msg {
98 struct list_head link;
99 u32 size;
100 u32 msg[] __counted_by(size);
101};
102
103enum { CTB_SEND = 0, CTB_RECV = 1 };
104
105enum { CTB_OWNER_HOST = 0 };
106
107/*
108 * Some H2G commands involve a synchronous response that the driver needs
109 * to wait for. In such cases, a timeout is required to prevent the driver
110 * from waiting forever in the case of an error (either no error response
111 * is defined in the protocol or something has died and requires a reset).
112 * The specific command may be defined as having a time bound response but
113 * the CT is a queue and that time guarantee only starts from the point
114 * when the command reaches the head of the queue and is processed by GuC.
115 *
116 * Ideally there would be a helper to report the progress of a given
117 * command through the CT. However, that would require a significant
118 * amount of work in the CT layer. In the meantime, provide a reasonable
119 * estimation of the worst case latency it should take for the entire
120 * queue to drain. And therefore, how long a caller should wait before
121 * giving up on their request. The current estimate is based on empirical
122 * measurement of a test that fills the buffer with context creation and
123 * destruction requests as they seem to be the slowest operation.
124 */
125long intel_guc_ct_max_queue_time_jiffies(void)
126{
127 /*
128 * A 4KB buffer full of context destroy commands takes a little
129 * over a second to process so bump that to 2s to be super safe.
130 */
131 return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K;
132}
133
134static void ct_receive_tasklet_func(struct tasklet_struct *t);
135static void ct_incoming_request_worker_func(struct work_struct *w);
136
137/**
138 * intel_guc_ct_init_early - Initialize CT state without requiring device access
139 * @ct: pointer to CT struct
140 */
141void intel_guc_ct_init_early(struct intel_guc_ct *ct)
142{
143 spin_lock_init(&ct->ctbs.send.lock);
144 spin_lock_init(&ct->ctbs.recv.lock);
145 spin_lock_init(&ct->requests.lock);
146 INIT_LIST_HEAD(list: &ct->requests.pending);
147 INIT_LIST_HEAD(list: &ct->requests.incoming);
148#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
149 INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func);
150#endif
151 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
152 tasklet_setup(t: &ct->receive_tasklet, callback: ct_receive_tasklet_func);
153 init_waitqueue_head(&ct->wq);
154}
155
156static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc)
157{
158 memset(s: desc, c: 0, n: sizeof(*desc));
159}
160
161static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb)
162{
163 u32 space;
164
165 ctb->broken = false;
166 ctb->tail = 0;
167 ctb->head = 0;
168 space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space;
169 atomic_set(v: &ctb->space, i: space);
170
171 guc_ct_buffer_desc_init(desc: ctb->desc);
172}
173
174static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,
175 struct guc_ct_buffer_desc *desc,
176 u32 *cmds, u32 size_in_bytes, u32 resv_space)
177{
178 GEM_BUG_ON(size_in_bytes % 4);
179
180 ctb->desc = desc;
181 ctb->cmds = cmds;
182 ctb->size = size_in_bytes / 4;
183 ctb->resv_space = resv_space / 4;
184
185 guc_ct_buffer_reset(ctb);
186}
187
188static int guc_action_control_ctb(struct intel_guc *guc, u32 control)
189{
190 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
191 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
192 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
193 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB),
194 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control),
195 };
196 int ret;
197
198 GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE);
199
200 /* CT control must go over MMIO */
201 ret = intel_guc_send_mmio(guc, action: request, ARRAY_SIZE(request), NULL, response_buf_size: 0);
202
203 return ret > 0 ? -EPROTO : ret;
204}
205
206static int ct_control_enable(struct intel_guc_ct *ct, bool enable)
207{
208 int err;
209
210 err = guc_action_control_ctb(guc: ct_to_guc(ct), control: enable ?
211 GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE);
212 if (unlikely(err))
213 CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n",
214 str_enable_disable(enable), ERR_PTR(err));
215
216 return err;
217}
218
219static int ct_register_buffer(struct intel_guc_ct *ct, bool send,
220 u32 desc_addr, u32 buff_addr, u32 size)
221{
222 int err;
223
224 err = intel_guc_self_cfg64(guc: ct_to_guc(ct), key: send ?
225 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY :
226 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
227 value: desc_addr);
228 if (unlikely(err))
229 goto failed;
230
231 err = intel_guc_self_cfg64(guc: ct_to_guc(ct), key: send ?
232 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY :
233 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
234 value: buff_addr);
235 if (unlikely(err))
236 goto failed;
237
238 err = intel_guc_self_cfg32(guc: ct_to_guc(ct), key: send ?
239 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY :
240 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
241 value: size);
242 if (unlikely(err))
243failed:
244 CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n",
245 send ? "SEND" : "RECV", ERR_PTR(err));
246
247 return err;
248}
249
250/**
251 * intel_guc_ct_init - Init buffer-based communication
252 * @ct: pointer to CT struct
253 *
254 * Allocate memory required for buffer-based communication.
255 *
256 * Return: 0 on success, a negative errno code on failure.
257 */
258int intel_guc_ct_init(struct intel_guc_ct *ct)
259{
260 struct intel_guc *guc = ct_to_guc(ct);
261 struct guc_ct_buffer_desc *desc;
262 u32 blob_size;
263 u32 cmds_size;
264 u32 resv_space;
265 void *blob;
266 u32 *cmds;
267 int err;
268
269 err = i915_inject_probe_error(guc_to_i915(guc), -ENXIO);
270 if (err)
271 return err;
272
273 GEM_BUG_ON(ct->vma);
274
275 blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
276 err = intel_guc_allocate_and_map_vma(guc, size: blob_size, out_vma: &ct->vma, out_vaddr: &blob);
277 if (unlikely(err)) {
278 CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",
279 blob_size, ERR_PTR(err));
280 return err;
281 }
282
283 CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);
284
285 /* store pointers to desc and cmds for send ctb */
286 desc = blob;
287 cmds = blob + 2 * CTB_DESC_SIZE;
288 cmds_size = CTB_H2G_BUFFER_SIZE;
289 resv_space = 0;
290 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send",
291 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
292 resv_space);
293
294 guc_ct_buffer_init(ctb: &ct->ctbs.send, desc, cmds, size_in_bytes: cmds_size, resv_space);
295
296 /* store pointers to desc and cmds for recv ctb */
297 desc = blob + CTB_DESC_SIZE;
298 cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;
299 cmds_size = CTB_G2H_BUFFER_SIZE;
300 resv_space = G2H_ROOM_BUFFER_SIZE;
301 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv",
302 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
303 resv_space);
304
305 guc_ct_buffer_init(ctb: &ct->ctbs.recv, desc, cmds, size_in_bytes: cmds_size, resv_space);
306
307 return 0;
308}
309
310/**
311 * intel_guc_ct_fini - Fini buffer-based communication
312 * @ct: pointer to CT struct
313 *
314 * Deallocate memory required for buffer-based communication.
315 */
316void intel_guc_ct_fini(struct intel_guc_ct *ct)
317{
318 GEM_BUG_ON(ct->enabled);
319
320 tasklet_kill(t: &ct->receive_tasklet);
321 i915_vma_unpin_and_release(p_vma: &ct->vma, I915_VMA_RELEASE_MAP);
322 memset(s: ct, c: 0, n: sizeof(*ct));
323}
324
325/**
326 * intel_guc_ct_enable - Enable buffer based command transport.
327 * @ct: pointer to CT struct
328 *
329 * Return: 0 on success, a negative errno code on failure.
330 */
331int intel_guc_ct_enable(struct intel_guc_ct *ct)
332{
333 struct intel_guc *guc = ct_to_guc(ct);
334 u32 base, desc, cmds, size;
335 void *blob;
336 int err;
337
338 GEM_BUG_ON(ct->enabled);
339
340 /* vma should be already allocated and map'ed */
341 GEM_BUG_ON(!ct->vma);
342 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));
343 base = intel_guc_ggtt_offset(guc, vma: ct->vma);
344
345 /* blob should start with send descriptor */
346 blob = __px_vaddr(p: ct->vma->obj);
347 GEM_BUG_ON(blob != ct->ctbs.send.desc);
348
349 /* (re)initialize descriptors */
350 guc_ct_buffer_reset(ctb: &ct->ctbs.send);
351 guc_ct_buffer_reset(ctb: &ct->ctbs.recv);
352
353 /*
354 * Register both CT buffers starting with RECV buffer.
355 * Descriptors are in first half of the blob.
356 */
357 desc = base + ptrdiff(a: ct->ctbs.recv.desc, b: blob);
358 cmds = base + ptrdiff(a: ct->ctbs.recv.cmds, b: blob);
359 size = ct->ctbs.recv.size * 4;
360 err = ct_register_buffer(ct, send: false, desc_addr: desc, buff_addr: cmds, size);
361 if (unlikely(err))
362 goto err_out;
363
364 desc = base + ptrdiff(a: ct->ctbs.send.desc, b: blob);
365 cmds = base + ptrdiff(a: ct->ctbs.send.cmds, b: blob);
366 size = ct->ctbs.send.size * 4;
367 err = ct_register_buffer(ct, send: true, desc_addr: desc, buff_addr: cmds, size);
368 if (unlikely(err))
369 goto err_out;
370
371 err = ct_control_enable(ct, enable: true);
372 if (unlikely(err))
373 goto err_out;
374
375 ct->enabled = true;
376 ct->stall_time = KTIME_MAX;
377#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
378 ct->dead_ct_reported = false;
379 ct->dead_ct_reason = CT_DEAD_ALIVE;
380#endif
381
382 return 0;
383
384err_out:
385 CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));
386 CT_DEAD(ct, SETUP);
387 return err;
388}
389
390/**
391 * intel_guc_ct_disable - Disable buffer based command transport.
392 * @ct: pointer to CT struct
393 */
394void intel_guc_ct_disable(struct intel_guc_ct *ct)
395{
396 struct intel_guc *guc = ct_to_guc(ct);
397
398 GEM_BUG_ON(!ct->enabled);
399
400 ct->enabled = false;
401
402 if (intel_guc_is_fw_running(guc)) {
403 ct_control_enable(ct, enable: false);
404 }
405}
406
407#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
408static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action)
409{
410 unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found);
411#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
412 unsigned long entries[SZ_32];
413 unsigned int n;
414
415 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
416
417 /* May be called under spinlock, so avoid sleeping */
418 ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT);
419#endif
420 ct->requests.lost_and_found[lost].fence = fence;
421 ct->requests.lost_and_found[lost].action = action;
422}
423#endif
424
425static u32 ct_get_next_fence(struct intel_guc_ct *ct)
426{
427 /* For now it's trivial */
428 return ++ct->requests.last_fence;
429}
430
431static int ct_write(struct intel_guc_ct *ct,
432 const u32 *action,
433 u32 len /* in dwords */,
434 u32 fence, u32 flags)
435{
436 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
437 struct guc_ct_buffer_desc *desc = ctb->desc;
438 u32 tail = ctb->tail;
439 u32 size = ctb->size;
440 u32 header;
441 u32 hxg;
442 u32 type;
443 u32 *cmds = ctb->cmds;
444 unsigned int i;
445
446 if (unlikely(desc->status))
447 goto corrupted;
448
449 GEM_BUG_ON(tail > size);
450
451#ifdef CONFIG_DRM_I915_DEBUG_GUC
452 if (unlikely(tail != READ_ONCE(desc->tail))) {
453 CT_ERROR(ct, "Tail was modified %u != %u\n",
454 desc->tail, tail);
455 desc->status |= GUC_CTB_STATUS_MISMATCH;
456 goto corrupted;
457 }
458 if (unlikely(READ_ONCE(desc->head) >= size)) {
459 CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
460 desc->head, size);
461 desc->status |= GUC_CTB_STATUS_OVERFLOW;
462 goto corrupted;
463 }
464#endif
465
466 /*
467 * dw0: CT header (including fence)
468 * dw1: HXG header (including action code)
469 * dw2+: action data
470 */
471 header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
472 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
473 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence);
474
475 type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST :
476 GUC_HXG_TYPE_REQUEST;
477 hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |
478 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION |
479 GUC_HXG_REQUEST_MSG_0_DATA0, action[0]);
480
481 CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n",
482 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]);
483
484 cmds[tail] = header;
485 tail = (tail + 1) % size;
486
487 cmds[tail] = hxg;
488 tail = (tail + 1) % size;
489
490 for (i = 1; i < len; i++) {
491 cmds[tail] = action[i];
492 tail = (tail + 1) % size;
493 }
494 GEM_BUG_ON(tail > size);
495
496#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
497 ct_track_lost_and_found(ct, fence,
498 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0]));
499#endif
500
501 /*
502 * make sure H2G buffer update and LRC tail update (if this triggering a
503 * submission) are visible before updating the descriptor tail
504 */
505 intel_guc_write_barrier(guc: ct_to_guc(ct));
506
507 /* update local copies */
508 ctb->tail = tail;
509 GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN);
510 atomic_sub(i: len + GUC_CTB_HDR_LEN, v: &ctb->space);
511
512 /* now update descriptor */
513 WRITE_ONCE(desc->tail, tail);
514
515 return 0;
516
517corrupted:
518 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
519 desc->head, desc->tail, desc->status);
520 CT_DEAD(ct, WRITE);
521 ctb->broken = true;
522 return -EPIPE;
523}
524
525/**
526 * wait_for_ct_request_update - Wait for CT request state update.
527 * @ct: pointer to CT
528 * @req: pointer to pending request
529 * @status: placeholder for status
530 *
531 * For each sent request, GuC shall send back CT response message.
532 * Our message handler will update status of tracked request once
533 * response message with given fence is received. Wait here and
534 * check for valid response status value.
535 *
536 * Return:
537 * * 0 response received (status is valid)
538 * * -ETIMEDOUT no response within hardcoded timeout
539 */
540static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)
541{
542 int err;
543 bool ct_enabled;
544
545 /*
546 * Fast commands should complete in less than 10us, so sample quickly
547 * up to that length of time, then switch to a slower sleep-wait loop.
548 * No GuC command should ever take longer than 10ms but many GuC
549 * commands can be inflight at time, so use a 1s timeout on the slower
550 * sleep-wait loop.
551 */
552#define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10
553#define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000
554#define done \
555 (!(ct_enabled = intel_guc_ct_enabled(ct)) || \
556 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
557 GUC_HXG_ORIGIN_GUC)
558 err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);
559 if (err)
560 err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);
561#undef done
562 if (!ct_enabled)
563 err = -ENODEV;
564
565 *status = req->status;
566 return err;
567}
568
569#define GUC_CTB_TIMEOUT_MS 1500
570static inline bool ct_deadlocked(struct intel_guc_ct *ct)
571{
572 long timeout = GUC_CTB_TIMEOUT_MS;
573 bool ret = ktime_ms_delta(later: ktime_get(), earlier: ct->stall_time) > timeout;
574
575 if (unlikely(ret)) {
576 struct guc_ct_buffer_desc *send = ct->ctbs.send.desc;
577 struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc;
578
579 CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n",
580 ktime_ms_delta(ktime_get(), ct->stall_time),
581 send->status, recv->status);
582 CT_ERROR(ct, "H2G Space: %u (Bytes)\n",
583 atomic_read(&ct->ctbs.send.space) * 4);
584 CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head);
585 CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail);
586 CT_ERROR(ct, "G2H Space: %u (Bytes)\n",
587 atomic_read(&ct->ctbs.recv.space) * 4);
588 CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head);
589 CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail);
590
591 CT_DEAD(ct, DEADLOCK);
592 ct->ctbs.send.broken = true;
593 }
594
595 return ret;
596}
597
598static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw)
599{
600 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
601
602 /*
603 * We leave a certain amount of space in the G2H CTB buffer for
604 * unexpected G2H CTBs (e.g. logging, engine hang, etc...)
605 */
606 return !g2h_len_dw || atomic_read(v: &ctb->space) >= g2h_len_dw;
607}
608
609static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
610{
611 lockdep_assert_held(&ct->ctbs.send.lock);
612
613 GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw));
614
615 if (g2h_len_dw)
616 atomic_sub(i: g2h_len_dw, v: &ct->ctbs.recv.space);
617}
618
619static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
620{
621 atomic_add(i: g2h_len_dw, v: &ct->ctbs.recv.space);
622}
623
624static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw)
625{
626 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
627 struct guc_ct_buffer_desc *desc = ctb->desc;
628 u32 head;
629 u32 space;
630
631 if (atomic_read(v: &ctb->space) >= len_dw)
632 return true;
633
634 head = READ_ONCE(desc->head);
635 if (unlikely(head > ctb->size)) {
636 CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
637 head, ctb->size);
638 desc->status |= GUC_CTB_STATUS_OVERFLOW;
639 ctb->broken = true;
640 CT_DEAD(ct, H2G_HAS_ROOM);
641 return false;
642 }
643
644 space = CIRC_SPACE(ctb->tail, head, ctb->size);
645 atomic_set(v: &ctb->space, i: space);
646
647 return space >= len_dw;
648}
649
650static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw)
651{
652 bool h2g = h2g_has_room(ct, len_dw: h2g_dw);
653 bool g2h = g2h_has_room(ct, g2h_len_dw: g2h_dw);
654
655 lockdep_assert_held(&ct->ctbs.send.lock);
656
657 if (unlikely(!h2g || !g2h)) {
658 if (ct->stall_time == KTIME_MAX)
659 ct->stall_time = ktime_get();
660
661 /* Be paranoid and kick G2H tasklet to free credits */
662 if (!g2h)
663 tasklet_hi_schedule(t: &ct->receive_tasklet);
664
665 if (unlikely(ct_deadlocked(ct)))
666 return -EPIPE;
667 else
668 return -EBUSY;
669 }
670
671 ct->stall_time = KTIME_MAX;
672 return 0;
673}
674
675#define G2H_LEN_DW(f) ({ \
676 typeof(f) f_ = (f); \
677 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \
678 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \
679 GUC_CTB_HXG_MSG_MIN_LEN : 0; \
680})
681static int ct_send_nb(struct intel_guc_ct *ct,
682 const u32 *action,
683 u32 len,
684 u32 flags)
685{
686 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
687 unsigned long spin_flags;
688 u32 g2h_len_dw = G2H_LEN_DW(flags);
689 u32 fence;
690 int ret;
691
692 spin_lock_irqsave(&ctb->lock, spin_flags);
693
694 ret = has_room_nb(ct, h2g_dw: len + GUC_CTB_HDR_LEN, g2h_dw: g2h_len_dw);
695 if (unlikely(ret))
696 goto out;
697
698 fence = ct_get_next_fence(ct);
699 ret = ct_write(ct, action, len, fence, flags);
700 if (unlikely(ret))
701 goto out;
702
703 g2h_reserve_space(ct, g2h_len_dw);
704 intel_guc_notify(guc: ct_to_guc(ct));
705
706out:
707 spin_unlock_irqrestore(lock: &ctb->lock, flags: spin_flags);
708
709 return ret;
710}
711
712static int ct_send(struct intel_guc_ct *ct,
713 const u32 *action,
714 u32 len,
715 u32 *response_buf,
716 u32 response_buf_size,
717 u32 *status)
718{
719 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
720 struct ct_request request;
721 unsigned long flags;
722 unsigned int sleep_period_ms = 1;
723 bool send_again;
724 u32 fence;
725 int err;
726
727 GEM_BUG_ON(!ct->enabled);
728 GEM_BUG_ON(!len);
729 GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN);
730 GEM_BUG_ON(!response_buf && response_buf_size);
731 might_sleep();
732
733resend:
734 send_again = false;
735
736 /*
737 * We use a lazy spin wait loop here as we believe that if the CT
738 * buffers are sized correctly the flow control condition should be
739 * rare. Reserving the maximum size in the G2H credits as we don't know
740 * how big the response is going to be.
741 */
742retry:
743 spin_lock_irqsave(&ctb->lock, flags);
744 if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) ||
745 !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) {
746 if (ct->stall_time == KTIME_MAX)
747 ct->stall_time = ktime_get();
748 spin_unlock_irqrestore(lock: &ctb->lock, flags);
749
750 if (unlikely(ct_deadlocked(ct)))
751 return -EPIPE;
752
753 if (msleep_interruptible(msecs: sleep_period_ms))
754 return -EINTR;
755 sleep_period_ms = sleep_period_ms << 1;
756
757 goto retry;
758 }
759
760 ct->stall_time = KTIME_MAX;
761
762 fence = ct_get_next_fence(ct);
763 request.fence = fence;
764 request.status = 0;
765 request.response_len = response_buf_size;
766 request.response_buf = response_buf;
767
768 spin_lock(lock: &ct->requests.lock);
769 list_add_tail(new: &request.link, head: &ct->requests.pending);
770 spin_unlock(lock: &ct->requests.lock);
771
772 err = ct_write(ct, action, len, fence, flags: 0);
773 g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
774
775 spin_unlock_irqrestore(lock: &ctb->lock, flags);
776
777 if (unlikely(err))
778 goto unlink;
779
780 intel_guc_notify(guc: ct_to_guc(ct));
781
782 err = wait_for_ct_request_update(ct, req: &request, status);
783 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
784 if (unlikely(err)) {
785 if (err == -ENODEV)
786 /* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.
787 * In this case, output is debug rather than error info
788 */
789 CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",
790 action[0], request.fence);
791 else
792 CT_ERROR(ct, "No response for request %#x (fence %u)\n",
793 action[0], request.fence);
794 goto unlink;
795 }
796
797 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
798 CT_DEBUG(ct, "retrying request %#x (%u)\n", *action,
799 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status));
800 send_again = true;
801 goto unlink;
802 }
803
804 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) {
805 err = -EIO;
806 goto unlink;
807 }
808
809 if (response_buf) {
810 /* There shall be no data in the status */
811 WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status));
812 /* Return actual response len */
813 err = request.response_len;
814 } else {
815 /* There shall be no response payload */
816 WARN_ON(request.response_len);
817 /* Return data decoded from the status dword */
818 err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status);
819 }
820
821unlink:
822 spin_lock_irqsave(&ct->requests.lock, flags);
823 list_del(entry: &request.link);
824 spin_unlock_irqrestore(lock: &ct->requests.lock, flags);
825
826 if (unlikely(send_again))
827 goto resend;
828
829 return err;
830}
831
832/*
833 * Command Transport (CT) buffer based GuC send function.
834 */
835int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
836 u32 *response_buf, u32 response_buf_size, u32 flags)
837{
838 u32 status = ~0; /* undefined */
839 int ret;
840
841 if (unlikely(!ct->enabled)) {
842 struct intel_guc *guc = ct_to_guc(ct);
843 struct intel_uc *uc = container_of(guc, struct intel_uc, guc);
844
845 WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action);
846 return -ENODEV;
847 }
848
849 if (unlikely(ct->ctbs.send.broken))
850 return -EPIPE;
851
852 if (flags & INTEL_GUC_CT_SEND_NB)
853 return ct_send_nb(ct, action, len, flags);
854
855 ret = ct_send(ct, action, len, response_buf, response_buf_size, status: &status);
856 if (unlikely(ret < 0)) {
857 if (ret != -ENODEV)
858 CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
859 action[0], ERR_PTR(ret), status);
860 } else if (unlikely(ret)) {
861 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
862 action[0], ret, ret);
863 }
864
865 return ret;
866}
867
868static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)
869{
870 struct ct_incoming_msg *msg;
871
872 msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC);
873 if (msg)
874 msg->size = num_dwords;
875 return msg;
876}
877
878static void ct_free_msg(struct ct_incoming_msg *msg)
879{
880 kfree(objp: msg);
881}
882
883/*
884 * Return: number available remaining dwords to read (0 if empty)
885 * or a negative error code on failure
886 */
887static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
888{
889 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
890 struct guc_ct_buffer_desc *desc = ctb->desc;
891 u32 head = ctb->head;
892 u32 tail = READ_ONCE(desc->tail);
893 u32 size = ctb->size;
894 u32 *cmds = ctb->cmds;
895 s32 available;
896 unsigned int len;
897 unsigned int i;
898 u32 header;
899
900 if (unlikely(ctb->broken))
901 return -EPIPE;
902
903 if (unlikely(desc->status)) {
904 u32 status = desc->status;
905
906 if (status & GUC_CTB_STATUS_UNUSED) {
907 /*
908 * Potentially valid if a CLIENT_RESET request resulted in
909 * contexts/engines being reset. But should never happen as
910 * no contexts should be active when CLIENT_RESET is sent.
911 */
912 CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");
913 status &= ~GUC_CTB_STATUS_UNUSED;
914 }
915
916 if (status)
917 goto corrupted;
918 }
919
920 GEM_BUG_ON(head > size);
921
922#ifdef CONFIG_DRM_I915_DEBUG_GUC
923 if (unlikely(head != READ_ONCE(desc->head))) {
924 CT_ERROR(ct, "Head was modified %u != %u\n",
925 desc->head, head);
926 desc->status |= GUC_CTB_STATUS_MISMATCH;
927 goto corrupted;
928 }
929#endif
930 if (unlikely(tail >= size)) {
931 CT_ERROR(ct, "Invalid tail offset %u >= %u)\n",
932 tail, size);
933 desc->status |= GUC_CTB_STATUS_OVERFLOW;
934 goto corrupted;
935 }
936
937 /* tail == head condition indicates empty */
938 available = tail - head;
939 if (unlikely(available == 0)) {
940 *msg = NULL;
941 return 0;
942 }
943
944 /* beware of buffer wrap case */
945 if (unlikely(available < 0))
946 available += size;
947 CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size);
948 GEM_BUG_ON(available < 0);
949
950 header = cmds[head];
951 head = (head + 1) % size;
952
953 /* message len with header */
954 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN;
955 if (unlikely(len > (u32)available)) {
956 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
957 4, &header,
958 4 * (head + available - 1 > size ?
959 size - head : available - 1), &cmds[head],
960 4 * (head + available - 1 > size ?
961 available - 1 - size + head : 0), &cmds[0]);
962 desc->status |= GUC_CTB_STATUS_UNDERFLOW;
963 goto corrupted;
964 }
965
966 *msg = ct_alloc_msg(num_dwords: len);
967 if (!*msg) {
968 CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",
969 4, &header,
970 4 * (head + available - 1 > size ?
971 size - head : available - 1), &cmds[head],
972 4 * (head + available - 1 > size ?
973 available - 1 - size + head : 0), &cmds[0]);
974 return available;
975 }
976
977 (*msg)->msg[0] = header;
978
979 for (i = 1; i < len; i++) {
980 (*msg)->msg[i] = cmds[head];
981 head = (head + 1) % size;
982 }
983 CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);
984
985 /* update local copies */
986 ctb->head = head;
987
988 /* now update descriptor */
989 WRITE_ONCE(desc->head, head);
990
991 intel_guc_write_barrier(guc: ct_to_guc(ct));
992
993 return available - len;
994
995corrupted:
996 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
997 desc->head, desc->tail, desc->status);
998 ctb->broken = true;
999 CT_DEAD(ct, READ);
1000 return -EPIPE;
1001}
1002
1003#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
1004static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1005{
1006 unsigned int n;
1007 char *buf = NULL;
1008 bool found = false;
1009
1010 lockdep_assert_held(&ct->requests.lock);
1011
1012 for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) {
1013 if (ct->requests.lost_and_found[n].fence != fence)
1014 continue;
1015 found = true;
1016
1017#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
1018 buf = kmalloc(SZ_4K, GFP_NOWAIT);
1019 if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack,
1020 buf, SZ_4K, 0)) {
1021 CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s",
1022 fence, ct->requests.lost_and_found[n].action, buf);
1023 break;
1024 }
1025#endif
1026 CT_ERROR(ct, "Fence %u was used by action %#04x\n",
1027 fence, ct->requests.lost_and_found[n].action);
1028 break;
1029 }
1030 kfree(buf);
1031 return found;
1032}
1033#else
1034static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1035{
1036 return false;
1037}
1038#endif
1039
1040static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)
1041{
1042 u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]);
1043 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]);
1044 const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN];
1045 const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN];
1046 u32 datalen = len - GUC_HXG_MSG_MIN_LEN;
1047 struct ct_request *req;
1048 unsigned long flags;
1049 bool found = false;
1050 int err = 0;
1051
1052 GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN);
1053 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC);
1054 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS &&
1055 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY &&
1056 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE);
1057
1058 CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]);
1059
1060 spin_lock_irqsave(&ct->requests.lock, flags);
1061 list_for_each_entry(req, &ct->requests.pending, link) {
1062 if (unlikely(fence != req->fence)) {
1063 CT_DEBUG(ct, "request %u awaits response\n",
1064 req->fence);
1065 continue;
1066 }
1067 if (unlikely(datalen > req->response_len)) {
1068 CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",
1069 req->fence, datalen, req->response_len);
1070 datalen = min(datalen, req->response_len);
1071 err = -EMSGSIZE;
1072 }
1073 if (datalen)
1074 memcpy(to: req->response_buf, from: data, len: 4 * datalen);
1075 req->response_len = datalen;
1076 WRITE_ONCE(req->status, hxg[0]);
1077 found = true;
1078 break;
1079 }
1080
1081#ifdef CONFIG_DRM_I915_SELFTEST
1082 if (!found && ct_to_guc(ct)->fast_response_selftest) {
1083 CT_DEBUG(ct, "Assuming unsolicited response due to FAST_REQUEST selftest\n");
1084 ct_to_guc(ct)->fast_response_selftest++;
1085 found = true;
1086 }
1087#endif
1088
1089 if (!found) {
1090 CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n",
1091 len, hxg[0], fence, ct->requests.last_fence);
1092 if (!ct_check_lost_and_found(ct, fence)) {
1093 list_for_each_entry(req, &ct->requests.pending, link)
1094 CT_ERROR(ct, "request %u awaits response\n",
1095 req->fence);
1096 }
1097 err = -ENOKEY;
1098 }
1099 spin_unlock_irqrestore(lock: &ct->requests.lock, flags);
1100
1101 if (unlikely(err))
1102 return err;
1103
1104 ct_free_msg(msg: response);
1105 return 0;
1106}
1107
1108static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1109{
1110 struct intel_guc *guc = ct_to_guc(ct);
1111 const u32 *hxg;
1112 const u32 *payload;
1113 u32 hxg_len, action, len;
1114 int ret;
1115
1116 hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1117 hxg_len = request->size - GUC_CTB_MSG_MIN_LEN;
1118 payload = &hxg[GUC_HXG_MSG_MIN_LEN];
1119 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1120 len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1121
1122 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
1123
1124 switch (action) {
1125 case INTEL_GUC_ACTION_DEFAULT:
1126 ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
1127 break;
1128 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1129 ret = intel_guc_deregister_done_process_msg(guc, msg: payload,
1130 len);
1131 break;
1132 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1133 ret = intel_guc_sched_done_process_msg(guc, msg: payload, len);
1134 break;
1135 case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1136 ret = intel_guc_context_reset_process_msg(guc, msg: payload, len);
1137 break;
1138 case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1139 ret = intel_guc_error_capture_process_msg(guc, msg: payload, len);
1140 if (unlikely(ret))
1141 CT_ERROR(ct, "error capture notification failed %x %*ph\n",
1142 action, 4 * len, payload);
1143 break;
1144 case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1145 ret = intel_guc_engine_failure_process_msg(guc, msg: payload, len);
1146 break;
1147 case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1148 intel_guc_log_handle_flush_event(log: &guc->log);
1149 ret = 0;
1150 break;
1151 case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1152 case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:
1153 ret = intel_guc_crash_process_msg(guc, action);
1154 break;
1155 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1156 ret = intel_guc_tlb_invalidation_done(guc, payload, len);
1157 break;
1158 default:
1159 ret = -EOPNOTSUPP;
1160 break;
1161 }
1162
1163 if (unlikely(ret)) {
1164 CT_ERROR(ct, "Failed to process request %04x (%pe)\n",
1165 action, ERR_PTR(ret));
1166 return ret;
1167 }
1168
1169 ct_free_msg(msg: request);
1170 return 0;
1171}
1172
1173static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
1174{
1175 unsigned long flags;
1176 struct ct_incoming_msg *request;
1177 bool done;
1178 int err;
1179
1180 spin_lock_irqsave(&ct->requests.lock, flags);
1181 request = list_first_entry_or_null(&ct->requests.incoming,
1182 struct ct_incoming_msg, link);
1183 if (request)
1184 list_del(entry: &request->link);
1185 done = !!list_empty(head: &ct->requests.incoming);
1186 spin_unlock_irqrestore(lock: &ct->requests.lock, flags);
1187
1188 if (!request)
1189 return true;
1190
1191 err = ct_process_request(ct, request);
1192 if (unlikely(err)) {
1193 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1194 ERR_PTR(err), 4 * request->size, request->msg);
1195 CT_DEAD(ct, PROCESS_FAILED);
1196 ct_free_msg(msg: request);
1197 }
1198
1199 return done;
1200}
1201
1202static void ct_incoming_request_worker_func(struct work_struct *w)
1203{
1204 struct intel_guc_ct *ct =
1205 container_of(w, struct intel_guc_ct, requests.worker);
1206 bool done;
1207
1208 do {
1209 done = ct_process_incoming_requests(ct);
1210 } while (!done);
1211}
1212
1213static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1214{
1215 const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1216 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1217 unsigned long flags;
1218
1219 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT);
1220
1221 /*
1222 * Adjusting the space must be done in IRQ or deadlock can occur as the
1223 * CTB processing in the below workqueue can send CTBs which creates a
1224 * circular dependency if the space was returned there.
1225 */
1226 switch (action) {
1227 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1228 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1229 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1230 g2h_release_space(ct, g2h_len_dw: request->size);
1231 }
1232
1233 /*
1234 * TLB invalidation responses must be handled immediately as processing
1235 * of other G2H notifications may be blocked by an invalidation request.
1236 */
1237 if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE)
1238 return ct_process_request(ct, request);
1239
1240 spin_lock_irqsave(&ct->requests.lock, flags);
1241 list_add_tail(new: &request->link, head: &ct->requests.incoming);
1242 spin_unlock_irqrestore(lock: &ct->requests.lock, flags);
1243
1244 queue_work(wq: system_unbound_wq, work: &ct->requests.worker);
1245 return 0;
1246}
1247
1248static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1249{
1250 u32 origin, type;
1251 u32 *hxg;
1252 int err;
1253
1254 if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN))
1255 return -EBADMSG;
1256
1257 hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN];
1258
1259 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1260 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1261 err = -EPROTO;
1262 goto failed;
1263 }
1264
1265 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1266 switch (type) {
1267 case GUC_HXG_TYPE_EVENT:
1268 err = ct_handle_event(ct, request: msg);
1269 break;
1270 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1271 case GUC_HXG_TYPE_RESPONSE_FAILURE:
1272 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1273 err = ct_handle_response(ct, response: msg);
1274 break;
1275 default:
1276 err = -EOPNOTSUPP;
1277 }
1278
1279 if (unlikely(err)) {
1280failed:
1281 CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n",
1282 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg);
1283 }
1284 return err;
1285}
1286
1287static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1288{
1289 u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]);
1290 int err;
1291
1292 if (format == GUC_CTB_FORMAT_HXG)
1293 err = ct_handle_hxg(ct, msg);
1294 else
1295 err = -EOPNOTSUPP;
1296
1297 if (unlikely(err)) {
1298 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1299 ERR_PTR(err), 4 * msg->size, msg->msg);
1300 ct_free_msg(msg);
1301 }
1302}
1303
1304/*
1305 * Return: number available remaining dwords to read (0 if empty)
1306 * or a negative error code on failure
1307 */
1308static int ct_receive(struct intel_guc_ct *ct)
1309{
1310 struct ct_incoming_msg *msg = NULL;
1311 unsigned long flags;
1312 int ret;
1313
1314 spin_lock_irqsave(&ct->ctbs.recv.lock, flags);
1315 ret = ct_read(ct, msg: &msg);
1316 spin_unlock_irqrestore(lock: &ct->ctbs.recv.lock, flags);
1317 if (ret < 0)
1318 return ret;
1319
1320 if (msg)
1321 ct_handle_msg(ct, msg);
1322
1323 return ret;
1324}
1325
1326static void ct_try_receive_message(struct intel_guc_ct *ct)
1327{
1328 int ret;
1329
1330 if (GEM_WARN_ON(!ct->enabled))
1331 return;
1332
1333 ret = ct_receive(ct);
1334 if (ret > 0)
1335 tasklet_hi_schedule(t: &ct->receive_tasklet);
1336}
1337
1338static void ct_receive_tasklet_func(struct tasklet_struct *t)
1339{
1340 struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);
1341
1342 ct_try_receive_message(ct);
1343}
1344
1345/*
1346 * When we're communicating with the GuC over CT, GuC uses events
1347 * to notify us about new messages being posted on the RECV buffer.
1348 */
1349void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
1350{
1351 if (unlikely(!ct->enabled)) {
1352 WARN(1, "Unexpected GuC event received while CT disabled!\n");
1353 return;
1354 }
1355
1356 ct_try_receive_message(ct);
1357}
1358
1359void intel_guc_ct_print_info(struct intel_guc_ct *ct,
1360 struct drm_printer *p)
1361{
1362 drm_printf(p, f: "CT %s\n", str_enabled_disabled(v: ct->enabled));
1363
1364 if (!ct->enabled)
1365 return;
1366
1367 drm_printf(p, f: "H2G Space: %u\n",
1368 atomic_read(v: &ct->ctbs.send.space) * 4);
1369 drm_printf(p, f: "Head: %u\n",
1370 ct->ctbs.send.desc->head);
1371 drm_printf(p, f: "Tail: %u\n",
1372 ct->ctbs.send.desc->tail);
1373 drm_printf(p, f: "G2H Space: %u\n",
1374 atomic_read(v: &ct->ctbs.recv.space) * 4);
1375 drm_printf(p, f: "Head: %u\n",
1376 ct->ctbs.recv.desc->head);
1377 drm_printf(p, f: "Tail: %u\n",
1378 ct->ctbs.recv.desc->tail);
1379}
1380
1381#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
1382static void ct_dead_ct_worker_func(struct work_struct *w)
1383{
1384 struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker);
1385 struct intel_guc *guc = ct_to_guc(ct);
1386
1387 if (ct->dead_ct_reported)
1388 return;
1389
1390 if (i915_error_injected())
1391 return;
1392
1393 ct->dead_ct_reported = true;
1394
1395 guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason);
1396 intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U);
1397}
1398#endif
1399