1// SPDX-License-Identifier: GPL-2.0
2/*
3 * xHCI host controller driver
4 *
5 * Copyright (C) 2008 Intel Corp.
6 *
7 * Author: Sarah Sharp
8 * Some code borrowed from the Linux EHCI driver.
9 */
10
11/*
12 * Ring initialization rules:
13 * 1. Each segment is initialized to zero, except for link TRBs.
14 * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or
15 * Consumer Cycle State (CCS), depending on ring function.
16 * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
17 *
18 * Ring behavior rules:
19 * 1. A ring is empty if enqueue == dequeue. This means there will always be at
20 * least one free TRB in the ring. This is useful if you want to turn that
21 * into a link TRB and expand the ring.
22 * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
23 * link TRB, then load the pointer with the address in the link TRB. If the
24 * link TRB had its toggle bit set, you may need to update the ring cycle
25 * state (see cycle bit rules). You may have to do this multiple times
26 * until you reach a non-link TRB.
27 * 3. A ring is full if enqueue++ (for the definition of increment above)
28 * equals the dequeue pointer.
29 *
30 * Cycle bit rules:
31 * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
32 * in a link TRB, it must toggle the ring cycle state.
33 * 2. When a producer increments an enqueue pointer and encounters a toggle bit
34 * in a link TRB, it must toggle the ring cycle state.
35 *
36 * Producer rules:
37 * 1. Check if ring is full before you enqueue.
38 * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
39 * Update enqueue pointer between each write (which may update the ring
40 * cycle state).
41 * 3. Notify consumer. If SW is producer, it rings the doorbell for command
42 * and endpoint rings. If HC is the producer for the event ring,
43 * and it generates an interrupt according to interrupt modulation rules.
44 *
45 * Consumer rules:
46 * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state,
47 * the TRB is owned by the consumer.
48 * 2. Update dequeue pointer (which may update the ring cycle state) and
49 * continue processing TRBs until you reach a TRB which is not owned by you.
50 * 3. Notify the producer. SW is the consumer for the event ring, and it
51 * updates event ring dequeue pointer. HC is the consumer for the command and
52 * endpoint rings; it generates events on the event ring for these.
53 */
54
55#include <linux/jiffies.h>
56#include <linux/scatterlist.h>
57#include <linux/slab.h>
58#include <linux/string_choices.h>
59#include <linux/dma-mapping.h>
60#include "xhci.h"
61#include "xhci-trace.h"
62
63static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
64 u32 field1, u32 field2,
65 u32 field3, u32 field4, bool command_must_succeed);
66
67/*
68 * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
69 * address of the TRB.
70 */
71dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
72 union xhci_trb *trb)
73{
74 unsigned long segment_offset;
75
76 if (!seg || !trb || trb < seg->trbs)
77 return 0;
78 /* offset in TRBs */
79 segment_offset = trb - seg->trbs;
80 if (segment_offset >= TRBS_PER_SEGMENT)
81 return 0;
82 return seg->dma + (segment_offset * sizeof(*trb));
83}
84
85static bool trb_is_noop(union xhci_trb *trb)
86{
87 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
88}
89
90static bool trb_is_link(union xhci_trb *trb)
91{
92 return TRB_TYPE_LINK_LE32(trb->link.control);
93}
94
95static bool last_trb_on_seg(struct xhci_segment *seg, union xhci_trb *trb)
96{
97 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
98}
99
100static bool last_trb_on_ring(struct xhci_ring *ring,
101 struct xhci_segment *seg, union xhci_trb *trb)
102{
103 return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
104}
105
106static bool link_trb_toggles_cycle(union xhci_trb *trb)
107{
108 return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
109}
110
111static bool last_td_in_urb(struct xhci_td *td)
112{
113 struct urb_priv *urb_priv = td->urb->hcpriv;
114
115 return urb_priv->num_tds_done == urb_priv->num_tds;
116}
117
118static bool unhandled_event_trb(struct xhci_ring *ring)
119{
120 return ((le32_to_cpu(ring->dequeue->event_cmd.flags) & TRB_CYCLE) ==
121 ring->cycle_state);
122}
123
124static void inc_td_cnt(struct urb *urb)
125{
126 struct urb_priv *urb_priv = urb->hcpriv;
127
128 urb_priv->num_tds_done++;
129}
130
131static void trb_to_noop(union xhci_trb *trb, u32 noop_type)
132{
133 if (trb_is_link(trb)) {
134 /* unchain chained link TRBs */
135 trb->link.control &= cpu_to_le32(~TRB_CHAIN);
136 } else {
137 trb->generic.field[0] = 0;
138 trb->generic.field[1] = 0;
139 trb->generic.field[2] = 0;
140 /* Preserve only the cycle bit of this TRB */
141 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
142 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
143 }
144}
145
146/* Updates trb to point to the next TRB in the ring, and updates seg if the next
147 * TRB is in a new segment. This does not skip over link TRBs, and it does not
148 * effect the ring dequeue or enqueue pointers.
149 */
150static void next_trb(struct xhci_segment **seg,
151 union xhci_trb **trb)
152{
153 if (trb_is_link(trb: *trb) || last_trb_on_seg(seg: *seg, trb: *trb)) {
154 *seg = (*seg)->next;
155 *trb = ((*seg)->trbs);
156 } else {
157 (*trb)++;
158 }
159}
160
161/*
162 * See Cycle bit rules. SW is the consumer for the event ring only.
163 */
164void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
165{
166 unsigned int link_trb_count = 0;
167
168 /* event ring doesn't have link trbs, check for last trb */
169 if (ring->type == TYPE_EVENT) {
170 if (!last_trb_on_seg(seg: ring->deq_seg, trb: ring->dequeue)) {
171 ring->dequeue++;
172 return;
173 }
174 if (last_trb_on_ring(ring, seg: ring->deq_seg, trb: ring->dequeue))
175 ring->cycle_state ^= 1;
176 ring->deq_seg = ring->deq_seg->next;
177 ring->dequeue = ring->deq_seg->trbs;
178
179 trace_xhci_inc_deq(ring);
180
181 return;
182 }
183
184 /* All other rings have link trbs */
185 if (!trb_is_link(trb: ring->dequeue)) {
186 if (last_trb_on_seg(seg: ring->deq_seg, trb: ring->dequeue))
187 xhci_warn(xhci, "Missing link TRB at end of segment\n");
188 else
189 ring->dequeue++;
190 }
191
192 while (trb_is_link(trb: ring->dequeue)) {
193 ring->deq_seg = ring->deq_seg->next;
194 ring->dequeue = ring->deq_seg->trbs;
195
196 trace_xhci_inc_deq(ring);
197
198 if (link_trb_count++ > ring->num_segs) {
199 xhci_warn(xhci, "Ring is an endless link TRB loop\n");
200 break;
201 }
202 }
203 return;
204}
205
206/*
207 * If enqueue points at a link TRB, follow links until an ordinary TRB is reached.
208 * Toggle the cycle bit of passed link TRBs and optionally chain them.
209 */
210static void inc_enq_past_link(struct xhci_hcd *xhci, struct xhci_ring *ring, u32 chain)
211{
212 unsigned int link_trb_count = 0;
213
214 while (trb_is_link(trb: ring->enqueue)) {
215
216 /*
217 * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
218 * set, but other sections talk about dealing with the chain bit set. This was
219 * fixed in the 0.96 specification errata, but we have to assume that all 0.95
220 * xHCI hardware can't handle the chain bit being cleared on a link TRB.
221 *
222 * On 0.95 and some 0.96 HCs the chain bit is set once at segment initalization
223 * and never changed here. On all others, modify it as requested by the caller.
224 */
225 if (!xhci_link_chain_quirk(xhci, type: ring->type)) {
226 ring->enqueue->link.control &= cpu_to_le32(~TRB_CHAIN);
227 ring->enqueue->link.control |= cpu_to_le32(chain);
228 }
229
230 /* Give this link TRB to the hardware */
231 wmb();
232 ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
233
234 /* Toggle the cycle bit after the last ring segment. */
235 if (link_trb_toggles_cycle(trb: ring->enqueue))
236 ring->cycle_state ^= 1;
237
238 ring->enq_seg = ring->enq_seg->next;
239 ring->enqueue = ring->enq_seg->trbs;
240
241 trace_xhci_inc_enq(ring);
242
243 if (link_trb_count++ > ring->num_segs) {
244 xhci_warn(xhci, "Link TRB loop at enqueue\n");
245 break;
246 }
247 }
248}
249
250/*
251 * See Cycle bit rules. SW is the consumer for the event ring only.
252 *
253 * If we've just enqueued a TRB that is in the middle of a TD (meaning the
254 * chain bit is set), then set the chain bit in all the following link TRBs.
255 * If we've enqueued the last TRB in a TD, make sure the following link TRBs
256 * have their chain bit cleared (so that each Link TRB is a separate TD).
257 *
258 * @more_trbs_coming: Will you enqueue more TRBs before calling
259 * prepare_transfer()?
260 */
261static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
262 bool more_trbs_coming)
263{
264 u32 chain;
265
266 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
267
268 if (last_trb_on_seg(seg: ring->enq_seg, trb: ring->enqueue)) {
269 xhci_err(xhci, "Tried to move enqueue past ring segment\n");
270 return;
271 }
272
273 ring->enqueue++;
274
275 /*
276 * If we are in the middle of a TD or the caller plans to enqueue more
277 * TDs as one transfer (eg. control), traverse any link TRBs right now.
278 * Otherwise, enqueue can stay on a link until the next prepare_ring().
279 * This avoids enqueue entering deq_seg and simplifies ring expansion.
280 */
281 if (trb_is_link(trb: ring->enqueue) && (chain || more_trbs_coming))
282 inc_enq_past_link(xhci, ring, chain);
283}
284
285/*
286 * If the suspect DMA address is a TRB in this TD, this function returns that
287 * TRB's segment. Otherwise it returns 0.
288 */
289static struct xhci_segment *trb_in_td(struct xhci_td *td, dma_addr_t suspect_dma)
290{
291 dma_addr_t start_dma;
292 dma_addr_t end_seg_dma;
293 dma_addr_t end_trb_dma;
294 struct xhci_segment *cur_seg;
295
296 start_dma = xhci_trb_virt_to_dma(seg: td->start_seg, trb: td->start_trb);
297 cur_seg = td->start_seg;
298
299 do {
300 if (start_dma == 0)
301 return NULL;
302 /* We may get an event for a Link TRB in the middle of a TD */
303 end_seg_dma = xhci_trb_virt_to_dma(seg: cur_seg,
304 trb: &cur_seg->trbs[TRBS_PER_SEGMENT - 1]);
305 /* If the end TRB isn't in this segment, this is set to 0 */
306 end_trb_dma = xhci_trb_virt_to_dma(seg: cur_seg, trb: td->end_trb);
307
308 if (end_trb_dma > 0) {
309 /* The end TRB is in this segment, so suspect should be here */
310 if (start_dma <= end_trb_dma) {
311 if (suspect_dma >= start_dma && suspect_dma <= end_trb_dma)
312 return cur_seg;
313 } else {
314 /* Case for one segment with
315 * a TD wrapped around to the top
316 */
317 if ((suspect_dma >= start_dma &&
318 suspect_dma <= end_seg_dma) ||
319 (suspect_dma >= cur_seg->dma &&
320 suspect_dma <= end_trb_dma))
321 return cur_seg;
322 }
323 return NULL;
324 }
325 /* Might still be somewhere in this segment */
326 if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
327 return cur_seg;
328
329 cur_seg = cur_seg->next;
330 start_dma = xhci_trb_virt_to_dma(seg: cur_seg, trb: &cur_seg->trbs[0]);
331 } while (cur_seg != td->start_seg);
332
333 return NULL;
334}
335
336/*
337 * Return number of free normal TRBs from enqueue to dequeue pointer on ring.
338 * Not counting an assumed link TRB at end of each TRBS_PER_SEGMENT sized segment.
339 * Only for transfer and command rings where driver is the producer, not for
340 * event rings.
341 */
342static unsigned int xhci_num_trbs_free(struct xhci_ring *ring)
343{
344 struct xhci_segment *enq_seg = ring->enq_seg;
345 union xhci_trb *enq = ring->enqueue;
346 union xhci_trb *last_on_seg;
347 unsigned int free = 0;
348 int i = 0;
349
350 /* Ring might be empty even if enq != deq if enq is left on a link trb */
351 if (trb_is_link(trb: enq)) {
352 enq_seg = enq_seg->next;
353 enq = enq_seg->trbs;
354 }
355
356 /* Empty ring, common case, don't walk the segments */
357 if (enq == ring->dequeue)
358 return ring->num_segs * (TRBS_PER_SEGMENT - 1);
359
360 do {
361 if (ring->deq_seg == enq_seg && ring->dequeue >= enq)
362 return free + (ring->dequeue - enq);
363 last_on_seg = &enq_seg->trbs[TRBS_PER_SEGMENT - 1];
364 free += last_on_seg - enq;
365 enq_seg = enq_seg->next;
366 enq = enq_seg->trbs;
367 } while (i++ < ring->num_segs);
368
369 return free;
370}
371
372/*
373 * Check to see if there's room to enqueue num_trbs on the ring and make sure
374 * enqueue pointer will not advance into dequeue segment. See rules above.
375 * return number of new segments needed to ensure this.
376 */
377
378static unsigned int xhci_ring_expansion_needed(struct xhci_hcd *xhci, struct xhci_ring *ring,
379 unsigned int num_trbs)
380{
381 struct xhci_segment *seg;
382 int trbs_past_seg;
383 int enq_used;
384 int new_segs;
385
386 enq_used = ring->enqueue - ring->enq_seg->trbs;
387
388 /* how many trbs will be queued past the enqueue segment? */
389 trbs_past_seg = enq_used + num_trbs - (TRBS_PER_SEGMENT - 1);
390
391 /*
392 * Consider expanding the ring already if num_trbs fills the current
393 * segment (i.e. trbs_past_seg == 0), not only when num_trbs goes into
394 * the next segment. Avoids confusing full ring with special empty ring
395 * case below
396 */
397 if (trbs_past_seg < 0)
398 return 0;
399
400 /* Empty ring special case, enqueue stuck on link trb while dequeue advanced */
401 if (trb_is_link(trb: ring->enqueue) && ring->enq_seg->next->trbs == ring->dequeue)
402 return 0;
403
404 new_segs = 1 + (trbs_past_seg / (TRBS_PER_SEGMENT - 1));
405 seg = ring->enq_seg;
406
407 while (new_segs > 0) {
408 seg = seg->next;
409 if (seg == ring->deq_seg) {
410 xhci_dbg(xhci, "Adding %d trbs requires expanding ring by %d segments\n",
411 num_trbs, new_segs);
412 return new_segs;
413 }
414 new_segs--;
415 }
416
417 return 0;
418}
419
420/* Ring the host controller doorbell after placing a command on the ring */
421void xhci_ring_cmd_db(struct xhci_hcd *xhci)
422{
423 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
424 return;
425
426 xhci_dbg(xhci, "// Ding dong!\n");
427
428 trace_xhci_ring_host_doorbell(slot: 0, DB_VALUE_HOST);
429
430 writel(DB_VALUE_HOST, addr: &xhci->dba->doorbell[0]);
431 /* Flush PCI posted writes */
432 readl(addr: &xhci->dba->doorbell[0]);
433}
434
435static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci)
436{
437 return mod_delayed_work(wq: system_wq, dwork: &xhci->cmd_timer,
438 delay: msecs_to_jiffies(m: xhci->current_cmd->timeout_ms));
439}
440
441static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci)
442{
443 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command,
444 cmd_list);
445}
446
447/*
448 * Turn all commands on command ring with status set to "aborted" to no-op trbs.
449 * If there are other commands waiting then restart the ring and kick the timer.
450 * This must be called with command ring stopped and xhci->lock held.
451 */
452static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
453 struct xhci_command *cur_cmd)
454{
455 struct xhci_command *i_cmd;
456
457 /* Turn all aborted commands in list to no-ops, then restart */
458 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) {
459
460 if (i_cmd->status != COMP_COMMAND_ABORTED)
461 continue;
462
463 i_cmd->status = COMP_COMMAND_RING_STOPPED;
464
465 xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
466 i_cmd->command_trb);
467
468 trb_to_noop(trb: i_cmd->command_trb, TRB_CMD_NOOP);
469
470 /*
471 * caller waiting for completion is called when command
472 * completion event is received for these no-op commands
473 */
474 }
475
476 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
477
478 /* ring command ring doorbell to restart the command ring */
479 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
480 !(xhci->xhc_state & XHCI_STATE_DYING)) {
481 xhci->current_cmd = cur_cmd;
482 if (cur_cmd)
483 xhci_mod_cmd_timer(xhci);
484 xhci_ring_cmd_db(xhci);
485 }
486}
487
488/* Must be called with xhci->lock held, releases and acquires lock back */
489static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
490{
491 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg;
492 union xhci_trb *new_deq = xhci->cmd_ring->dequeue;
493 u64 crcr;
494 int ret;
495
496 xhci_dbg(xhci, "Abort command ring\n");
497
498 reinit_completion(x: &xhci->cmd_ring_stop_completion);
499
500 /*
501 * The control bits like command stop, abort are located in lower
502 * dword of the command ring control register.
503 * Some controllers require all 64 bits to be written to abort the ring.
504 * Make sure the upper dword is valid, pointing to the next command,
505 * avoiding corrupting the command ring pointer in case the command ring
506 * is stopped by the time the upper dword is written.
507 */
508 next_trb(seg: &new_seg, trb: &new_deq);
509 if (trb_is_link(trb: new_deq))
510 next_trb(seg: &new_seg, trb: &new_deq);
511
512 crcr = xhci_trb_virt_to_dma(seg: new_seg, trb: new_deq);
513 xhci_write_64(xhci, val: crcr | CMD_RING_ABORT, regs: &xhci->op_regs->cmd_ring);
514
515 /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the
516 * completion of the Command Abort operation. If CRR is not negated in 5
517 * seconds then driver handles it as if host died (-ENODEV).
518 * In the future we should distinguish between -ENODEV and -ETIMEDOUT
519 * and try to recover a -ETIMEDOUT with a host controller reset.
520 */
521 ret = xhci_handshake(ptr: &xhci->op_regs->cmd_ring,
522 CMD_RING_RUNNING, done: 0, timeout_us: 5 * 1000 * 1000);
523 if (ret < 0) {
524 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret);
525 xhci_halt(xhci);
526 xhci_hc_died(xhci);
527 return ret;
528 }
529 /*
530 * Writing the CMD_RING_ABORT bit should cause a cmd completion event,
531 * however on some host hw the CMD_RING_RUNNING bit is correctly cleared
532 * but the completion event in never sent. Wait 2 secs (arbitrary
533 * number) to handle those cases after negation of CMD_RING_RUNNING.
534 */
535 spin_unlock_irqrestore(lock: &xhci->lock, flags);
536 ret = wait_for_completion_timeout(x: &xhci->cmd_ring_stop_completion,
537 timeout: msecs_to_jiffies(m: 2000));
538 spin_lock_irqsave(&xhci->lock, flags);
539 if (!ret) {
540 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n");
541 xhci_cleanup_command_queue(xhci);
542 } else {
543 xhci_handle_stopped_cmd_ring(xhci, cur_cmd: xhci_next_queued_cmd(xhci));
544 }
545 return 0;
546}
547
548void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
549 unsigned int slot_id,
550 unsigned int ep_index,
551 unsigned int stream_id)
552{
553 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
554 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
555 unsigned int ep_state = ep->ep_state;
556
557 /* Don't ring the doorbell for this endpoint if there are pending
558 * cancellations because we don't want to interrupt processing.
559 * We don't want to restart any stream rings if there's a set dequeue
560 * pointer command pending because the device can choose to start any
561 * stream once the endpoint is on the HW schedule.
562 */
563 if ((ep_state & EP_STOP_CMD_PENDING) || (ep_state & SET_DEQ_PENDING) ||
564 (ep_state & EP_HALTED) || (ep_state & EP_CLEARING_TT))
565 return;
566
567 trace_xhci_ring_ep_doorbell(slot: slot_id, DB_VALUE(ep_index, stream_id));
568
569 writel(DB_VALUE(ep_index, stream_id), addr: db_addr);
570 /* flush the write */
571 readl(addr: db_addr);
572}
573
574/* Ring the doorbell for any rings with pending URBs */
575static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
576 unsigned int slot_id,
577 unsigned int ep_index)
578{
579 unsigned int stream_id;
580 struct xhci_virt_ep *ep;
581
582 ep = &xhci->devs[slot_id]->eps[ep_index];
583
584 /* A ring has pending URBs if its TD list is not empty */
585 if (!(ep->ep_state & EP_HAS_STREAMS)) {
586 if (ep->ring && !(list_empty(head: &ep->ring->td_list)))
587 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id: 0);
588 return;
589 }
590
591 for (stream_id = 1; stream_id < ep->stream_info->num_streams;
592 stream_id++) {
593 struct xhci_stream_info *stream_info = ep->stream_info;
594 if (!list_empty(head: &stream_info->stream_rings[stream_id]->td_list))
595 xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
596 stream_id);
597 }
598}
599
600void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
601 unsigned int slot_id,
602 unsigned int ep_index)
603{
604 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
605}
606
607static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci,
608 unsigned int slot_id,
609 unsigned int ep_index)
610{
611 if (slot_id == 0 || slot_id >= MAX_HC_SLOTS) {
612 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id);
613 return NULL;
614 }
615 if (ep_index >= EP_CTX_PER_DEV) {
616 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index);
617 return NULL;
618 }
619 if (!xhci->devs[slot_id]) {
620 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id);
621 return NULL;
622 }
623
624 return &xhci->devs[slot_id]->eps[ep_index];
625}
626
627static struct xhci_ring *xhci_virt_ep_to_ring(struct xhci_hcd *xhci,
628 struct xhci_virt_ep *ep,
629 unsigned int stream_id)
630{
631 /* common case, no streams */
632 if (!(ep->ep_state & EP_HAS_STREAMS))
633 return ep->ring;
634
635 if (!ep->stream_info)
636 return NULL;
637
638 if (stream_id == 0 || stream_id >= ep->stream_info->num_streams) {
639 xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n",
640 stream_id, ep->vdev->slot_id, ep->ep_index);
641 return NULL;
642 }
643
644 return ep->stream_info->stream_rings[stream_id];
645}
646
647/* Get the right ring for the given slot_id, ep_index and stream_id.
648 * If the endpoint supports streams, boundary check the URB's stream ID.
649 * If the endpoint doesn't support streams, return the singular endpoint ring.
650 */
651struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
652 unsigned int slot_id, unsigned int ep_index,
653 unsigned int stream_id)
654{
655 struct xhci_virt_ep *ep;
656
657 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
658 if (!ep)
659 return NULL;
660
661 return xhci_virt_ep_to_ring(xhci, ep, stream_id);
662}
663
664
665/*
666 * Get the hw dequeue pointer xHC stopped on, either directly from the
667 * endpoint context, or if streams are in use from the stream context.
668 * The returned hw_dequeue contains the lowest four bits with cycle state
669 * and possbile stream context type.
670 */
671static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev,
672 unsigned int ep_index, unsigned int stream_id)
673{
674 struct xhci_ep_ctx *ep_ctx;
675 struct xhci_stream_ctx *st_ctx;
676 struct xhci_virt_ep *ep;
677
678 ep = &vdev->eps[ep_index];
679
680 if (ep->ep_state & EP_HAS_STREAMS) {
681 st_ctx = &ep->stream_info->stream_ctx_array[stream_id];
682 return le64_to_cpu(st_ctx->stream_ring);
683 }
684 ep_ctx = xhci_get_ep_ctx(xhci, ctx: vdev->out_ctx, ep_index);
685 return le64_to_cpu(ep_ctx->deq);
686}
687
688static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci,
689 unsigned int slot_id, unsigned int ep_index,
690 unsigned int stream_id, struct xhci_td *td)
691{
692 struct xhci_virt_device *dev = xhci->devs[slot_id];
693 struct xhci_virt_ep *ep = &dev->eps[ep_index];
694 struct xhci_ring *ep_ring;
695 struct xhci_command *cmd;
696 struct xhci_segment *new_seg;
697 union xhci_trb *new_deq;
698 int new_cycle;
699 dma_addr_t addr;
700 u64 hw_dequeue;
701 bool hw_dequeue_found = false;
702 bool td_last_trb_found = false;
703 u32 trb_sct = 0;
704 int ret;
705
706 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
707 ep_index, stream_id);
708 if (!ep_ring) {
709 xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n",
710 stream_id);
711 return -ENODEV;
712 }
713
714 hw_dequeue = xhci_get_hw_deq(xhci, vdev: dev, ep_index, stream_id) & TR_DEQ_PTR_MASK;
715 new_seg = ep_ring->deq_seg;
716 new_deq = ep_ring->dequeue;
717 new_cycle = le32_to_cpu(td->end_trb->generic.field[3]) & TRB_CYCLE;
718
719 /*
720 * Walk the ring until both the next TRB and hw_dequeue are found (don't
721 * move hw_dequeue back if it went forward due to a HW bug). Cycle state
722 * is loaded from a known good TRB, track later toggles to maintain it.
723 */
724 do {
725 if (!hw_dequeue_found && xhci_trb_virt_to_dma(seg: new_seg, trb: new_deq)
726 == (dma_addr_t)hw_dequeue) {
727 hw_dequeue_found = true;
728 if (td_last_trb_found)
729 break;
730 }
731 if (new_deq == td->end_trb)
732 td_last_trb_found = true;
733
734 if (td_last_trb_found && trb_is_link(trb: new_deq) &&
735 link_trb_toggles_cycle(trb: new_deq))
736 new_cycle ^= 0x1;
737
738 next_trb(seg: &new_seg, trb: &new_deq);
739
740 /* Search wrapped around, bail out */
741 if (new_deq == ep->ring->dequeue) {
742 xhci_err(xhci, "Error: Failed finding new dequeue state\n");
743 return -EINVAL;
744 }
745
746 } while (!hw_dequeue_found || !td_last_trb_found);
747
748 /* Don't update the ring cycle state for the producer (us). */
749 addr = xhci_trb_virt_to_dma(seg: new_seg, trb: new_deq);
750 if (addr == 0) {
751 xhci_warn(xhci, "Can't find dma of new dequeue ptr\n");
752 xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq);
753 return -EINVAL;
754 }
755
756 if ((ep->ep_state & SET_DEQ_PENDING)) {
757 xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n",
758 &addr);
759 return -EBUSY;
760 }
761
762 /* This function gets called from contexts where it cannot sleep */
763 cmd = xhci_alloc_command(xhci, allocate_completion: false, GFP_ATOMIC);
764 if (!cmd) {
765 xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr);
766 return -ENOMEM;
767 }
768
769 if (stream_id)
770 trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
771 ret = queue_command(xhci, cmd,
772 lower_32_bits(addr) | trb_sct | new_cycle,
773 upper_32_bits(addr),
774 STREAM_ID_FOR_TRB(stream_id), SLOT_ID_FOR_TRB(slot_id) |
775 EP_INDEX_FOR_TRB(ep_index) | TRB_TYPE(TRB_SET_DEQ), command_must_succeed: false);
776 if (ret < 0) {
777 xhci_free_command(xhci, command: cmd);
778 return ret;
779 }
780 ep->queued_deq_seg = new_seg;
781 ep->queued_deq_ptr = new_deq;
782
783 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_cancel_urb,
784 fmt: "Set TR Deq ptr 0x%llx, cycle %u\n", addr, new_cycle);
785
786 /* Stop the TD queueing code from ringing the doorbell until
787 * this command completes. The HC won't set the dequeue pointer
788 * if the ring is running, and ringing the doorbell starts the
789 * ring running.
790 */
791 ep->ep_state |= SET_DEQ_PENDING;
792 xhci_ring_cmd_db(xhci);
793 return 0;
794}
795
796/* flip_cycle means flip the cycle bit of all but the first and last TRB.
797 * (The last TRB actually points to the ring enqueue pointer, which is not part
798 * of this TD.) This is used to remove partially enqueued isoc TDs from a ring.
799 */
800static void td_to_noop(struct xhci_td *td, bool flip_cycle)
801{
802 struct xhci_segment *seg = td->start_seg;
803 union xhci_trb *trb = td->start_trb;
804
805 while (1) {
806 trb_to_noop(trb, TRB_TR_NOOP);
807
808 /* flip cycle if asked to */
809 if (flip_cycle && trb != td->start_trb && trb != td->end_trb)
810 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
811
812 if (trb == td->end_trb)
813 break;
814
815 next_trb(seg: &seg, trb: &trb);
816 }
817}
818
819static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
820 struct xhci_td *cur_td, int status)
821{
822 struct urb *urb = cur_td->urb;
823 struct urb_priv *urb_priv = urb->hcpriv;
824 struct usb_hcd *hcd = bus_to_hcd(bus: urb->dev->bus);
825
826 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
827 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
828 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
829 if (xhci->quirks & XHCI_AMD_PLL_FIX)
830 usb_amd_quirk_pll_enable();
831 }
832 }
833 xhci_urb_free_priv(urb_priv);
834 usb_hcd_unlink_urb_from_ep(hcd, urb);
835 trace_xhci_urb_giveback(urb);
836 usb_hcd_giveback_urb(hcd, urb, status);
837}
838
839static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
840 struct xhci_ring *ring, struct xhci_td *td)
841{
842 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
843 struct xhci_segment *seg = td->bounce_seg;
844 struct urb *urb = td->urb;
845 size_t len;
846
847 if (!ring || !seg || !urb)
848 return;
849
850 if (usb_urb_dir_out(urb)) {
851 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
852 DMA_TO_DEVICE);
853 return;
854 }
855
856 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
857 DMA_FROM_DEVICE);
858 /* for in transfers we need to copy the data from bounce to sg */
859 if (urb->num_sgs) {
860 len = sg_pcopy_from_buffer(sgl: urb->sg, nents: urb->num_sgs, buf: seg->bounce_buf,
861 buflen: seg->bounce_len, skip: seg->bounce_offs);
862 if (len != seg->bounce_len)
863 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n",
864 len, seg->bounce_len);
865 } else {
866 memcpy(to: urb->transfer_buffer + seg->bounce_offs, from: seg->bounce_buf,
867 len: seg->bounce_len);
868 }
869 seg->bounce_len = 0;
870 seg->bounce_offs = 0;
871}
872
873static void xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td,
874 struct xhci_ring *ep_ring, int status)
875{
876 struct urb *urb = NULL;
877
878 /* Clean up the endpoint's TD list */
879 urb = td->urb;
880
881 /* if a bounce buffer was used to align this td then unmap it */
882 xhci_unmap_td_bounce_buffer(xhci, ring: ep_ring, td);
883
884 /* Do one last check of the actual transfer length.
885 * If the host controller said we transferred more data than the buffer
886 * length, urb->actual_length will be a very big number (since it's
887 * unsigned). Play it safe and say we didn't transfer anything.
888 */
889 if (urb->actual_length > urb->transfer_buffer_length) {
890 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
891 urb->transfer_buffer_length, urb->actual_length);
892 urb->actual_length = 0;
893 status = 0;
894 }
895 /* TD might be removed from td_list if we are giving back a cancelled URB */
896 if (!list_empty(head: &td->td_list))
897 list_del_init(entry: &td->td_list);
898 /* Giving back a cancelled URB, or if a slated TD completed anyway */
899 if (!list_empty(head: &td->cancelled_td_list))
900 list_del_init(entry: &td->cancelled_td_list);
901
902 inc_td_cnt(urb);
903 /* Giveback the urb when all the tds are completed */
904 if (last_td_in_urb(td)) {
905 if ((urb->actual_length != urb->transfer_buffer_length &&
906 (urb->transfer_flags & URB_SHORT_NOT_OK)) ||
907 (status != 0 && !usb_endpoint_xfer_isoc(epd: &urb->ep->desc)))
908 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
909 urb, urb->actual_length,
910 urb->transfer_buffer_length, status);
911
912 /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
913 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
914 status = 0;
915 xhci_giveback_urb_in_irq(xhci, cur_td: td, status);
916 }
917}
918
919/* Give back previous TD and move on to the next TD. */
920static void xhci_dequeue_td(struct xhci_hcd *xhci, struct xhci_td *td, struct xhci_ring *ring,
921 u32 status)
922{
923 ring->dequeue = td->end_trb;
924 ring->deq_seg = td->end_seg;
925 inc_deq(xhci, ring);
926
927 xhci_td_cleanup(xhci, td, ep_ring: ring, status);
928}
929
930/* Complete the cancelled URBs we unlinked from td_list. */
931static void xhci_giveback_invalidated_tds(struct xhci_virt_ep *ep)
932{
933 struct xhci_ring *ring;
934 struct xhci_td *td, *tmp_td;
935
936 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list,
937 cancelled_td_list) {
938
939 ring = xhci_urb_to_transfer_ring(xhci: ep->xhci, urb: td->urb);
940
941 if (td->cancel_status == TD_CLEARED) {
942 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
943 __func__, td->urb);
944 xhci_td_cleanup(xhci: ep->xhci, td, ep_ring: ring, status: td->status);
945 } else {
946 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
947 __func__, td->urb, td->cancel_status);
948 }
949 if (ep->xhci->xhc_state & XHCI_STATE_DYING)
950 return;
951 }
952}
953
954static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id,
955 unsigned int ep_index, enum xhci_ep_reset_type reset_type)
956{
957 struct xhci_command *command;
958 int ret = 0;
959
960 command = xhci_alloc_command(xhci, allocate_completion: false, GFP_ATOMIC);
961 if (!command) {
962 ret = -ENOMEM;
963 goto done;
964 }
965
966 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n",
967 (reset_type == EP_HARD_RESET) ? "Hard" : "Soft",
968 ep_index, slot_id);
969
970 ret = xhci_queue_reset_ep(xhci, cmd: command, slot_id, ep_index, reset_type);
971done:
972 if (ret)
973 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n",
974 slot_id, ep_index, ret);
975 return ret;
976}
977
978static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci,
979 struct xhci_virt_ep *ep,
980 struct xhci_td *td,
981 enum xhci_ep_reset_type reset_type)
982{
983 unsigned int slot_id = ep->vdev->slot_id;
984 int err;
985
986 /*
987 * Avoid resetting endpoint if link is inactive. Can cause host hang.
988 * Device will be reset soon to recover the link so don't do anything
989 */
990 if (ep->vdev->flags & VDEV_PORT_ERROR)
991 return -ENODEV;
992
993 /* add td to cancelled list and let reset ep handler take care of it */
994 if (reset_type == EP_HARD_RESET) {
995 ep->ep_state |= EP_HARD_CLEAR_TOGGLE;
996 if (td && list_empty(head: &td->cancelled_td_list)) {
997 list_add_tail(new: &td->cancelled_td_list, head: &ep->cancelled_td_list);
998 td->cancel_status = TD_HALTED;
999 }
1000 }
1001
1002 if (ep->ep_state & EP_HALTED) {
1003 xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n",
1004 ep->ep_index);
1005 return 0;
1006 }
1007
1008 err = xhci_reset_halted_ep(xhci, slot_id, ep_index: ep->ep_index, reset_type);
1009 if (err)
1010 return err;
1011
1012 ep->ep_state |= EP_HALTED;
1013
1014 xhci_ring_cmd_db(xhci);
1015
1016 return 0;
1017}
1018
1019/*
1020 * Fix up the ep ring first, so HW stops executing cancelled TDs.
1021 * We have the xHCI lock, so nothing can modify this list until we drop it.
1022 * We're also in the event handler, so we can't get re-interrupted if another
1023 * Stop Endpoint command completes.
1024 *
1025 * only call this when ring is not in a running state
1026 */
1027
1028static int xhci_invalidate_cancelled_tds(struct xhci_virt_ep *ep)
1029{
1030 struct xhci_hcd *xhci;
1031 struct xhci_td *td = NULL;
1032 struct xhci_td *tmp_td = NULL;
1033 struct xhci_td *cached_td = NULL;
1034 struct xhci_ring *ring;
1035 u64 hw_deq;
1036 unsigned int slot_id = ep->vdev->slot_id;
1037 int err;
1038
1039 /*
1040 * This is not going to work if the hardware is changing its dequeue
1041 * pointers as we look at them. Completion handler will call us later.
1042 */
1043 if (ep->ep_state & SET_DEQ_PENDING)
1044 return 0;
1045
1046 xhci = ep->xhci;
1047
1048 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) {
1049 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_cancel_urb,
1050 fmt: "Removing canceled TD starting at 0x%llx (dma) in stream %u URB %p",
1051 (unsigned long long)xhci_trb_virt_to_dma(
1052 seg: td->start_seg, trb: td->start_trb),
1053 td->urb->stream_id, td->urb);
1054 list_del_init(entry: &td->td_list);
1055 ring = xhci_urb_to_transfer_ring(xhci, urb: td->urb);
1056 if (!ring) {
1057 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n",
1058 td->urb, td->urb->stream_id);
1059 continue;
1060 }
1061 /*
1062 * If a ring stopped on the TD we need to cancel then we have to
1063 * move the xHC endpoint ring dequeue pointer past this TD.
1064 * Rings halted due to STALL may show hw_deq is past the stalled
1065 * TD, but still require a set TR Deq command to flush xHC cache.
1066 */
1067 hw_deq = xhci_get_hw_deq(xhci, vdev: ep->vdev, ep_index: ep->ep_index,
1068 stream_id: td->urb->stream_id);
1069 hw_deq &= TR_DEQ_PTR_MASK;
1070
1071 if (td->cancel_status == TD_HALTED || trb_in_td(td, suspect_dma: hw_deq)) {
1072 switch (td->cancel_status) {
1073 case TD_CLEARED: /* TD is already no-op */
1074 case TD_CLEARING_CACHE: /* set TR deq command already queued */
1075 break;
1076 case TD_DIRTY: /* TD is cached, clear it */
1077 case TD_HALTED:
1078 case TD_CLEARING_CACHE_DEFERRED:
1079 if (cached_td) {
1080 if (cached_td->urb->stream_id != td->urb->stream_id) {
1081 /* Multiple streams case, defer move dq */
1082 xhci_dbg(xhci,
1083 "Move dq deferred: stream %u URB %p\n",
1084 td->urb->stream_id, td->urb);
1085 td->cancel_status = TD_CLEARING_CACHE_DEFERRED;
1086 break;
1087 }
1088
1089 /* Should never happen, but clear the TD if it does */
1090 xhci_warn(xhci,
1091 "Found multiple active URBs %p and %p in stream %u?\n",
1092 td->urb, cached_td->urb,
1093 td->urb->stream_id);
1094 td_to_noop(td: cached_td, flip_cycle: false);
1095 cached_td->cancel_status = TD_CLEARED;
1096 }
1097 td_to_noop(td, flip_cycle: false);
1098 td->cancel_status = TD_CLEARING_CACHE;
1099 cached_td = td;
1100 break;
1101 }
1102 } else {
1103 td_to_noop(td, flip_cycle: false);
1104 td->cancel_status = TD_CLEARED;
1105 }
1106 }
1107
1108 /* If there's no need to move the dequeue pointer then we're done */
1109 if (!cached_td)
1110 return 0;
1111
1112 err = xhci_move_dequeue_past_td(xhci, slot_id, ep_index: ep->ep_index,
1113 stream_id: cached_td->urb->stream_id,
1114 td: cached_td);
1115 if (err) {
1116 /* Failed to move past cached td, just set cached TDs to no-op */
1117 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) {
1118 /*
1119 * Deferred TDs need to have the deq pointer set after the above command
1120 * completes, so if that failed we just give up on all of them (and
1121 * complain loudly since this could cause issues due to caching).
1122 */
1123 if (td->cancel_status != TD_CLEARING_CACHE &&
1124 td->cancel_status != TD_CLEARING_CACHE_DEFERRED)
1125 continue;
1126 xhci_warn(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n",
1127 td->urb);
1128 td_to_noop(td, flip_cycle: false);
1129 td->cancel_status = TD_CLEARED;
1130 }
1131 }
1132 return 0;
1133}
1134
1135/*
1136 * Erase queued TDs from transfer ring(s) and give back those the xHC didn't
1137 * stop on. If necessary, queue commands to move the xHC off cancelled TDs it
1138 * stopped on. Those will be given back later when the commands complete.
1139 *
1140 * Call under xhci->lock on a stopped endpoint.
1141 */
1142void xhci_process_cancelled_tds(struct xhci_virt_ep *ep)
1143{
1144 xhci_invalidate_cancelled_tds(ep);
1145 xhci_giveback_invalidated_tds(ep);
1146}
1147
1148/*
1149 * Returns the TD the endpoint ring halted on.
1150 * Only call for non-running rings without streams.
1151 */
1152static struct xhci_td *find_halted_td(struct xhci_virt_ep *ep)
1153{
1154 struct xhci_td *td;
1155 u64 hw_deq;
1156
1157 if (!list_empty(head: &ep->ring->td_list)) { /* Not streams compatible */
1158 hw_deq = xhci_get_hw_deq(xhci: ep->xhci, vdev: ep->vdev, ep_index: ep->ep_index, stream_id: 0);
1159 hw_deq &= TR_DEQ_PTR_MASK;
1160 td = list_first_entry(&ep->ring->td_list, struct xhci_td, td_list);
1161 if (trb_in_td(td, suspect_dma: hw_deq))
1162 return td;
1163 }
1164 return NULL;
1165}
1166
1167/*
1168 * When we get a command completion for a Stop Endpoint Command, we need to
1169 * unlink any cancelled TDs from the ring. There are two ways to do that:
1170 *
1171 * 1. If the HW was in the middle of processing the TD that needs to be
1172 * cancelled, then we must move the ring's dequeue pointer past the last TRB
1173 * in the TD with a Set Dequeue Pointer Command.
1174 * 2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
1175 * bit cleared) so that the HW will skip over them.
1176 */
1177static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
1178 union xhci_trb *trb, u32 comp_code)
1179{
1180 unsigned int ep_index;
1181 struct xhci_virt_ep *ep;
1182 struct xhci_ep_ctx *ep_ctx;
1183 struct xhci_td *td = NULL;
1184 enum xhci_ep_reset_type reset_type;
1185 struct xhci_command *command;
1186 int err;
1187
1188 if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
1189 if (!xhci->devs[slot_id])
1190 xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n",
1191 slot_id);
1192 return;
1193 }
1194
1195 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1196 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1197 if (!ep)
1198 return;
1199
1200 ep_ctx = xhci_get_ep_ctx(xhci, ctx: ep->vdev->out_ctx, ep_index);
1201
1202 trace_xhci_handle_cmd_stop_ep(ctx: ep_ctx);
1203
1204 if (comp_code == COMP_CONTEXT_STATE_ERROR) {
1205 /*
1206 * If stop endpoint command raced with a halting endpoint we need to
1207 * reset the host side endpoint first.
1208 * If the TD we halted on isn't cancelled the TD should be given back
1209 * with a proper error code, and the ring dequeue moved past the TD.
1210 * If streams case we can't find hw_deq, or the TD we halted on so do a
1211 * soft reset.
1212 *
1213 * Proper error code is unknown here, it would be -EPIPE if device side
1214 * of enadpoit halted (aka STALL), and -EPROTO if not (transaction error)
1215 * We use -EPROTO, if device is stalled it should return a stall error on
1216 * next transfer, which then will return -EPIPE, and device side stall is
1217 * noted and cleared by class driver.
1218 */
1219 switch (GET_EP_CTX_STATE(ep_ctx)) {
1220 case EP_STATE_HALTED:
1221 xhci_dbg(xhci, "Stop ep completion raced with stall\n");
1222 /*
1223 * If the halt happened before Stop Endpoint failed, its transfer event
1224 * should have already been handled and Reset Endpoint should be pending.
1225 */
1226 if (ep->ep_state & EP_HALTED)
1227 goto reset_done;
1228
1229 if (ep->ep_state & EP_HAS_STREAMS) {
1230 reset_type = EP_SOFT_RESET;
1231 } else {
1232 reset_type = EP_HARD_RESET;
1233 td = find_halted_td(ep);
1234 if (td)
1235 td->status = -EPROTO;
1236 }
1237 /* reset ep, reset handler cleans up cancelled tds */
1238 err = xhci_handle_halted_endpoint(xhci, ep, td, reset_type);
1239 xhci_dbg(xhci, "Stop ep completion resetting ep, status %d\n", err);
1240 if (err)
1241 break;
1242reset_done:
1243 /* Reset EP handler will clean up cancelled TDs */
1244 ep->ep_state &= ~EP_STOP_CMD_PENDING;
1245 return;
1246 case EP_STATE_STOPPED:
1247 /*
1248 * Per xHCI 4.6.9, Stop Endpoint command on a Stopped
1249 * EP is a Context State Error, and EP stays Stopped.
1250 *
1251 * But maybe it failed on Halted, and somebody ran Reset
1252 * Endpoint later. EP state is now Stopped and EP_HALTED
1253 * still set because Reset EP handler will run after us.
1254 */
1255 if (ep->ep_state & EP_HALTED)
1256 break;
1257 /*
1258 * On some HCs EP state remains Stopped for some tens of
1259 * us to a few ms or more after a doorbell ring, and any
1260 * new Stop Endpoint fails without aborting the restart.
1261 * This handler may run quickly enough to still see this
1262 * Stopped state, but it will soon change to Running.
1263 *
1264 * Assume this bug on unexpected Stop Endpoint failures.
1265 * Keep retrying until the EP starts and stops again or
1266 * up to a timeout (a defective HC may never start, or a
1267 * driver bug may cause stopping an already stopped EP).
1268 */
1269 if (time_is_before_jiffies(ep->stop_time + msecs_to_jiffies(100)))
1270 break;
1271 fallthrough;
1272 case EP_STATE_RUNNING:
1273 /* Race, HW handled stop ep cmd before ep was running */
1274 xhci_dbg(xhci, "Stop ep completion ctx error, ctx_state %d\n",
1275 GET_EP_CTX_STATE(ep_ctx));
1276
1277 command = xhci_alloc_command(xhci, allocate_completion: false, GFP_ATOMIC);
1278 if (!command) {
1279 ep->ep_state &= ~EP_STOP_CMD_PENDING;
1280 return;
1281 }
1282 xhci_queue_stop_endpoint(xhci, cmd: command, slot_id, ep_index, suspend: 0);
1283 xhci_ring_cmd_db(xhci);
1284
1285 return;
1286 default:
1287 break;
1288 }
1289 }
1290
1291 /* will queue a set TR deq if stopped on a cancelled, uncleared TD */
1292 xhci_invalidate_cancelled_tds(ep);
1293 ep->ep_state &= ~EP_STOP_CMD_PENDING;
1294
1295 /* Otherwise ring the doorbell(s) to restart queued transfers */
1296 xhci_giveback_invalidated_tds(ep);
1297 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1298}
1299
1300static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
1301{
1302 struct xhci_td *cur_td;
1303 struct xhci_td *tmp;
1304
1305 list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) {
1306 list_del_init(entry: &cur_td->td_list);
1307
1308 if (!list_empty(head: &cur_td->cancelled_td_list))
1309 list_del_init(entry: &cur_td->cancelled_td_list);
1310
1311 xhci_unmap_td_bounce_buffer(xhci, ring, td: cur_td);
1312
1313 inc_td_cnt(urb: cur_td->urb);
1314 if (last_td_in_urb(td: cur_td))
1315 xhci_giveback_urb_in_irq(xhci, cur_td, status: -ESHUTDOWN);
1316 }
1317}
1318
1319static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
1320 int slot_id, int ep_index)
1321{
1322 struct xhci_td *cur_td;
1323 struct xhci_td *tmp;
1324 struct xhci_virt_ep *ep;
1325 struct xhci_ring *ring;
1326
1327 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1328 if (!ep)
1329 return;
1330
1331 if ((ep->ep_state & EP_HAS_STREAMS) ||
1332 (ep->ep_state & EP_GETTING_NO_STREAMS)) {
1333 int stream_id;
1334
1335 for (stream_id = 1; stream_id < ep->stream_info->num_streams;
1336 stream_id++) {
1337 ring = ep->stream_info->stream_rings[stream_id];
1338 if (!ring)
1339 continue;
1340
1341 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_cancel_urb,
1342 fmt: "Killing URBs for slot ID %u, ep index %u, stream %u",
1343 slot_id, ep_index, stream_id);
1344 xhci_kill_ring_urbs(xhci, ring);
1345 }
1346 } else {
1347 ring = ep->ring;
1348 if (!ring)
1349 return;
1350 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_cancel_urb,
1351 fmt: "Killing URBs for slot ID %u, ep index %u",
1352 slot_id, ep_index);
1353 xhci_kill_ring_urbs(xhci, ring);
1354 }
1355
1356 list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list,
1357 cancelled_td_list) {
1358 list_del_init(entry: &cur_td->cancelled_td_list);
1359 inc_td_cnt(urb: cur_td->urb);
1360
1361 if (last_td_in_urb(td: cur_td))
1362 xhci_giveback_urb_in_irq(xhci, cur_td, status: -ESHUTDOWN);
1363 }
1364}
1365
1366/*
1367 * host controller died, register read returns 0xffffffff
1368 * Complete pending commands, mark them ABORTED.
1369 * URBs need to be given back as usb core might be waiting with device locks
1370 * held for the URBs to finish during device disconnect, blocking host remove.
1371 *
1372 * Call with xhci->lock held.
1373 * lock is relased and re-acquired while giving back urb.
1374 */
1375void xhci_hc_died(struct xhci_hcd *xhci)
1376{
1377 bool notify;
1378 int i, j;
1379
1380 if (xhci->xhc_state & XHCI_STATE_DYING)
1381 return;
1382
1383 notify = !(xhci->xhc_state & XHCI_STATE_REMOVING);
1384 if (notify)
1385 xhci_err(xhci, "xHCI host controller not responding, assume dead\n");
1386 xhci->xhc_state |= XHCI_STATE_DYING;
1387
1388 xhci_cleanup_command_queue(xhci);
1389
1390 /* return any pending urbs, remove may be waiting for them */
1391 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
1392 if (!xhci->devs[i])
1393 continue;
1394 for (j = 0; j < 31; j++)
1395 xhci_kill_endpoint_urbs(xhci, slot_id: i, ep_index: j);
1396 }
1397
1398 /* inform usb core hc died if PCI remove isn't already handling it */
1399 if (notify)
1400 usb_hc_died(hcd: xhci_to_hcd(xhci));
1401}
1402
1403/*
1404 * When we get a completion for a Set Transfer Ring Dequeue Pointer command,
1405 * we need to clear the set deq pending flag in the endpoint ring state, so that
1406 * the TD queueing code can ring the doorbell again. We also need to ring the
1407 * endpoint doorbell to restart the ring, but only if there aren't more
1408 * cancellations pending.
1409 */
1410static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
1411 union xhci_trb *trb, u32 cmd_comp_code)
1412{
1413 unsigned int ep_index;
1414 unsigned int stream_id;
1415 struct xhci_ring *ep_ring;
1416 struct xhci_virt_ep *ep;
1417 struct xhci_ep_ctx *ep_ctx;
1418 struct xhci_slot_ctx *slot_ctx;
1419 struct xhci_stream_ctx *stream_ctx;
1420 struct xhci_td *td, *tmp_td;
1421
1422 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1423 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
1424 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1425 if (!ep)
1426 return;
1427
1428 ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id);
1429 if (!ep_ring) {
1430 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
1431 stream_id);
1432 /* XXX: Harmless??? */
1433 goto cleanup;
1434 }
1435
1436 ep_ctx = xhci_get_ep_ctx(xhci, ctx: ep->vdev->out_ctx, ep_index);
1437 slot_ctx = xhci_get_slot_ctx(xhci, ctx: ep->vdev->out_ctx);
1438 trace_xhci_handle_cmd_set_deq(ctx: slot_ctx);
1439 trace_xhci_handle_cmd_set_deq_ep(ctx: ep_ctx);
1440
1441 if (ep->ep_state & EP_HAS_STREAMS) {
1442 stream_ctx = &ep->stream_info->stream_ctx_array[stream_id];
1443 trace_xhci_handle_cmd_set_deq_stream(info: ep->stream_info, stream_id);
1444 }
1445
1446 if (cmd_comp_code != COMP_SUCCESS) {
1447 unsigned int ep_state;
1448 unsigned int slot_state;
1449
1450 switch (cmd_comp_code) {
1451 case COMP_TRB_ERROR:
1452 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
1453 break;
1454 case COMP_CONTEXT_STATE_ERROR:
1455 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
1456 ep_state = GET_EP_CTX_STATE(ep_ctx);
1457 slot_state = le32_to_cpu(slot_ctx->dev_state);
1458 slot_state = GET_SLOT_STATE(slot_state);
1459 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_cancel_urb,
1460 fmt: "Slot state = %u, EP state = %u",
1461 slot_state, ep_state);
1462 break;
1463 case COMP_SLOT_NOT_ENABLED_ERROR:
1464 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
1465 slot_id);
1466 break;
1467 default:
1468 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
1469 cmd_comp_code);
1470 break;
1471 }
1472 /* OK what do we do now? The endpoint state is hosed, and we
1473 * should never get to this point if the synchronization between
1474 * queueing, and endpoint state are correct. This might happen
1475 * if the device gets disconnected after we've finished
1476 * cancelling URBs, which might not be an error...
1477 */
1478 } else {
1479 u64 deq;
1480 /* 4.6.10 deq ptr is written to the stream ctx for streams */
1481 if (ep->ep_state & EP_HAS_STREAMS) {
1482 deq = le64_to_cpu(stream_ctx->stream_ring) & TR_DEQ_PTR_MASK;
1483
1484 /*
1485 * Cadence xHCI controllers store some endpoint state
1486 * information within Rsvd0 fields of Stream Endpoint
1487 * context. This field is not cleared during Set TR
1488 * Dequeue Pointer command which causes XDMA to skip
1489 * over transfer ring and leads to data loss on stream
1490 * pipe.
1491 * To fix this issue driver must clear Rsvd0 field.
1492 */
1493 if (xhci->quirks & XHCI_CDNS_SCTX_QUIRK) {
1494 stream_ctx->reserved[0] = 0;
1495 stream_ctx->reserved[1] = 0;
1496 }
1497 } else {
1498 deq = le64_to_cpu(ep_ctx->deq) & TR_DEQ_PTR_MASK;
1499 }
1500 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_cancel_urb,
1501 fmt: "Successful Set TR Deq Ptr cmd, deq = @%08llx", deq);
1502 if (xhci_trb_virt_to_dma(seg: ep->queued_deq_seg,
1503 trb: ep->queued_deq_ptr) == deq) {
1504 /* Update the ring's dequeue segment and dequeue pointer
1505 * to reflect the new position.
1506 */
1507 ep_ring->deq_seg = ep->queued_deq_seg;
1508 ep_ring->dequeue = ep->queued_deq_ptr;
1509 } else {
1510 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
1511 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
1512 ep->queued_deq_seg, ep->queued_deq_ptr);
1513 }
1514 }
1515 /* HW cached TDs cleared from cache, give them back */
1516 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list,
1517 cancelled_td_list) {
1518 ep_ring = xhci_urb_to_transfer_ring(xhci: ep->xhci, urb: td->urb);
1519 if (td->cancel_status == TD_CLEARING_CACHE) {
1520 td->cancel_status = TD_CLEARED;
1521 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
1522 __func__, td->urb);
1523 xhci_td_cleanup(xhci: ep->xhci, td, ep_ring, status: td->status);
1524 } else {
1525 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
1526 __func__, td->urb, td->cancel_status);
1527 }
1528 }
1529cleanup:
1530 ep->ep_state &= ~SET_DEQ_PENDING;
1531 ep->queued_deq_seg = NULL;
1532 ep->queued_deq_ptr = NULL;
1533
1534 /* Check for deferred or newly cancelled TDs */
1535 if (!list_empty(head: &ep->cancelled_td_list)) {
1536 xhci_dbg(ep->xhci, "%s: Pending TDs to clear, continuing with invalidation\n",
1537 __func__);
1538 xhci_invalidate_cancelled_tds(ep);
1539 /* Try to restart the endpoint if all is done */
1540 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1541 /* Start giving back any TDs invalidated above */
1542 xhci_giveback_invalidated_tds(ep);
1543 } else {
1544 /* Restart any rings with pending URBs */
1545 xhci_dbg(ep->xhci, "%s: All TDs cleared, ring doorbell\n", __func__);
1546 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1547 }
1548}
1549
1550static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1551 union xhci_trb *trb, u32 cmd_comp_code)
1552{
1553 struct xhci_virt_ep *ep;
1554 struct xhci_ep_ctx *ep_ctx;
1555 unsigned int ep_index;
1556
1557 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1558 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1559 if (!ep)
1560 return;
1561
1562 ep_ctx = xhci_get_ep_ctx(xhci, ctx: ep->vdev->out_ctx, ep_index);
1563 trace_xhci_handle_cmd_reset_ep(ctx: ep_ctx);
1564
1565 /* This command will only fail if the endpoint wasn't halted,
1566 * but we don't care.
1567 */
1568 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_reset_ep,
1569 fmt: "Ignoring reset ep completion code of %u", cmd_comp_code);
1570
1571 /* Cleanup cancelled TDs as ep is stopped. May queue a Set TR Deq cmd */
1572 xhci_invalidate_cancelled_tds(ep);
1573
1574 /* Clear our internal halted state */
1575 ep->ep_state &= ~EP_HALTED;
1576
1577 xhci_giveback_invalidated_tds(ep);
1578
1579 /* if this was a soft reset, then restart */
1580 if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP)
1581 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1582}
1583
1584static void xhci_handle_cmd_enable_slot(int slot_id, struct xhci_command *command,
1585 u32 cmd_comp_code)
1586{
1587 if (cmd_comp_code == COMP_SUCCESS)
1588 command->slot_id = slot_id;
1589 else
1590 command->slot_id = 0;
1591}
1592
1593static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id,
1594 u32 cmd_comp_code)
1595{
1596 struct xhci_virt_device *virt_dev;
1597 struct xhci_slot_ctx *slot_ctx;
1598
1599 virt_dev = xhci->devs[slot_id];
1600 if (!virt_dev)
1601 return;
1602
1603 slot_ctx = xhci_get_slot_ctx(xhci, ctx: virt_dev->out_ctx);
1604 trace_xhci_handle_cmd_disable_slot(ctx: slot_ctx);
1605
1606 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1607 /* Delete default control endpoint resources */
1608 xhci_free_device_endpoint_resources(xhci, virt_dev, drop_control_ep: true);
1609 if (cmd_comp_code == COMP_SUCCESS) {
1610 xhci->dcbaa->dev_context_ptrs[slot_id] = 0;
1611 xhci->devs[slot_id] = NULL;
1612 }
1613}
1614
1615static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id)
1616{
1617 struct xhci_virt_device *virt_dev;
1618 struct xhci_input_control_ctx *ctrl_ctx;
1619 struct xhci_ep_ctx *ep_ctx;
1620 unsigned int ep_index;
1621 u32 add_flags;
1622
1623 /*
1624 * Configure endpoint commands can come from the USB core configuration
1625 * or alt setting changes, or when streams were being configured.
1626 */
1627
1628 virt_dev = xhci->devs[slot_id];
1629 if (!virt_dev)
1630 return;
1631 ctrl_ctx = xhci_get_input_control_ctx(ctx: virt_dev->in_ctx);
1632 if (!ctrl_ctx) {
1633 xhci_warn(xhci, "Could not get input context, bad type.\n");
1634 return;
1635 }
1636
1637 add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1638
1639 /* Input ctx add_flags are the endpoint index plus one */
1640 ep_index = xhci_last_valid_endpoint(added_ctxs: add_flags) - 1;
1641
1642 ep_ctx = xhci_get_ep_ctx(xhci, ctx: virt_dev->out_ctx, ep_index);
1643 trace_xhci_handle_cmd_config_ep(ctx: ep_ctx);
1644
1645 return;
1646}
1647
1648static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id)
1649{
1650 struct xhci_virt_device *vdev;
1651 struct xhci_slot_ctx *slot_ctx;
1652
1653 vdev = xhci->devs[slot_id];
1654 if (!vdev)
1655 return;
1656 slot_ctx = xhci_get_slot_ctx(xhci, ctx: vdev->out_ctx);
1657 trace_xhci_handle_cmd_addr_dev(ctx: slot_ctx);
1658}
1659
1660static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id)
1661{
1662 struct xhci_virt_device *vdev;
1663 struct xhci_slot_ctx *slot_ctx;
1664
1665 vdev = xhci->devs[slot_id];
1666 if (!vdev) {
1667 xhci_warn(xhci, "Reset device command completion for disabled slot %u\n",
1668 slot_id);
1669 return;
1670 }
1671 slot_ctx = xhci_get_slot_ctx(xhci, ctx: vdev->out_ctx);
1672 trace_xhci_handle_cmd_reset_dev(ctx: slot_ctx);
1673
1674 xhci_dbg(xhci, "Completed reset device command.\n");
1675}
1676
1677static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
1678 struct xhci_event_cmd *event)
1679{
1680 if (!(xhci->quirks & XHCI_NEC_HOST)) {
1681 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
1682 return;
1683 }
1684 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_quirks,
1685 fmt: "NEC firmware version %2x.%02x",
1686 NEC_FW_MAJOR(le32_to_cpu(event->status)),
1687 NEC_FW_MINOR(le32_to_cpu(event->status)));
1688}
1689
1690static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 comp_code, u32 comp_param)
1691{
1692 list_del(entry: &cmd->cmd_list);
1693
1694 if (cmd->completion) {
1695 cmd->status = comp_code;
1696 cmd->comp_param = comp_param;
1697 complete(cmd->completion);
1698 } else {
1699 kfree(objp: cmd);
1700 }
1701}
1702
1703void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
1704{
1705 struct xhci_command *cur_cmd, *tmp_cmd;
1706 xhci->current_cmd = NULL;
1707 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
1708 xhci_complete_del_and_free_cmd(cmd: cur_cmd, COMP_COMMAND_ABORTED, comp_param: 0);
1709}
1710
1711void xhci_handle_command_timeout(struct work_struct *work)
1712{
1713 struct xhci_hcd *xhci;
1714 unsigned long flags;
1715 char str[XHCI_MSG_MAX];
1716 u64 hw_ring_state;
1717 u32 cmd_field3;
1718 u32 usbsts;
1719
1720 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer);
1721
1722 spin_lock_irqsave(&xhci->lock, flags);
1723
1724 /*
1725 * If timeout work is pending, or current_cmd is NULL, it means we
1726 * raced with command completion. Command is handled so just return.
1727 */
1728 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) {
1729 spin_unlock_irqrestore(lock: &xhci->lock, flags);
1730 return;
1731 }
1732
1733 cmd_field3 = le32_to_cpu(xhci->current_cmd->command_trb->generic.field[3]);
1734 usbsts = readl(addr: &xhci->op_regs->status);
1735 xhci_dbg(xhci, "Command timeout, USBSTS:%s\n", xhci_decode_usbsts(str, usbsts));
1736
1737 /* Bail out and tear down xhci if a stop endpoint command failed */
1738 if (TRB_FIELD_TO_TYPE(cmd_field3) == TRB_STOP_RING) {
1739 struct xhci_virt_ep *ep;
1740
1741 xhci_warn(xhci, "xHCI host not responding to stop endpoint command\n");
1742
1743 ep = xhci_get_virt_ep(xhci, TRB_TO_SLOT_ID(cmd_field3),
1744 TRB_TO_EP_INDEX(cmd_field3));
1745 if (ep)
1746 ep->ep_state &= ~EP_STOP_CMD_PENDING;
1747
1748 xhci_halt(xhci);
1749 xhci_hc_died(xhci);
1750 goto time_out_completed;
1751 }
1752
1753 /* mark this command to be cancelled */
1754 xhci->current_cmd->status = COMP_COMMAND_ABORTED;
1755
1756 /* Make sure command ring is running before aborting it */
1757 hw_ring_state = xhci_read_64(xhci, regs: &xhci->op_regs->cmd_ring);
1758 if (hw_ring_state == ~(u64)0) {
1759 xhci_hc_died(xhci);
1760 goto time_out_completed;
1761 }
1762
1763 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
1764 (hw_ring_state & CMD_RING_RUNNING)) {
1765 /* Prevent new doorbell, and start command abort */
1766 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
1767 xhci_dbg(xhci, "Command timeout\n");
1768 xhci_abort_cmd_ring(xhci, flags);
1769 goto time_out_completed;
1770 }
1771
1772 /* host removed. Bail out */
1773 if (xhci->xhc_state & XHCI_STATE_REMOVING) {
1774 xhci_dbg(xhci, "host removed, ring start fail?\n");
1775 xhci_cleanup_command_queue(xhci);
1776
1777 goto time_out_completed;
1778 }
1779
1780 /* command timeout on stopped ring, ring can't be aborted */
1781 xhci_dbg(xhci, "Command timeout on stopped ring\n");
1782 xhci_handle_stopped_cmd_ring(xhci, cur_cmd: xhci->current_cmd);
1783
1784time_out_completed:
1785 spin_unlock_irqrestore(lock: &xhci->lock, flags);
1786 return;
1787}
1788
1789static void handle_cmd_completion(struct xhci_hcd *xhci,
1790 struct xhci_event_cmd *event)
1791{
1792 unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1793 u32 status = le32_to_cpu(event->status);
1794 u64 cmd_dma;
1795 dma_addr_t cmd_dequeue_dma;
1796 u32 cmd_comp_code;
1797 union xhci_trb *cmd_trb;
1798 struct xhci_command *cmd;
1799 u32 cmd_type;
1800
1801 if (slot_id >= MAX_HC_SLOTS) {
1802 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id);
1803 return;
1804 }
1805
1806 cmd_dma = le64_to_cpu(event->cmd_trb);
1807 cmd_trb = xhci->cmd_ring->dequeue;
1808
1809 trace_xhci_handle_command(ring: xhci->cmd_ring, trb: &cmd_trb->generic, dma: cmd_dma);
1810
1811 cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status));
1812
1813 /* If CMD ring stopped we own the trbs between enqueue and dequeue */
1814 if (cmd_comp_code == COMP_COMMAND_RING_STOPPED) {
1815 complete_all(&xhci->cmd_ring_stop_completion);
1816 return;
1817 }
1818
1819 cmd_dequeue_dma = xhci_trb_virt_to_dma(seg: xhci->cmd_ring->deq_seg,
1820 trb: cmd_trb);
1821 /*
1822 * Check whether the completion event is for our internal kept
1823 * command.
1824 */
1825 if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) {
1826 xhci_warn(xhci,
1827 "ERROR mismatched command completion event\n");
1828 return;
1829 }
1830
1831 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list);
1832
1833 cancel_delayed_work(dwork: &xhci->cmd_timer);
1834
1835 if (cmd->command_trb != xhci->cmd_ring->dequeue) {
1836 xhci_err(xhci,
1837 "Command completion event does not match command\n");
1838 return;
1839 }
1840
1841 /*
1842 * Host aborted the command ring, check if the current command was
1843 * supposed to be aborted, otherwise continue normally.
1844 * The command ring is stopped now, but the xHC will issue a Command
1845 * Ring Stopped event which will cause us to restart it.
1846 */
1847 if (cmd_comp_code == COMP_COMMAND_ABORTED) {
1848 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
1849 if (cmd->status == COMP_COMMAND_ABORTED) {
1850 if (xhci->current_cmd == cmd)
1851 xhci->current_cmd = NULL;
1852 goto event_handled;
1853 }
1854 }
1855
1856 cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
1857 switch (cmd_type) {
1858 case TRB_ENABLE_SLOT:
1859 xhci_handle_cmd_enable_slot(slot_id, command: cmd, cmd_comp_code);
1860 break;
1861 case TRB_DISABLE_SLOT:
1862 xhci_handle_cmd_disable_slot(xhci, slot_id, cmd_comp_code);
1863 break;
1864 case TRB_CONFIG_EP:
1865 if (!cmd->completion)
1866 xhci_handle_cmd_config_ep(xhci, slot_id);
1867 break;
1868 case TRB_EVAL_CONTEXT:
1869 break;
1870 case TRB_ADDR_DEV:
1871 xhci_handle_cmd_addr_dev(xhci, slot_id);
1872 break;
1873 case TRB_STOP_RING:
1874 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1875 le32_to_cpu(cmd_trb->generic.field[3])));
1876 if (!cmd->completion)
1877 xhci_handle_cmd_stop_ep(xhci, slot_id, trb: cmd_trb,
1878 comp_code: cmd_comp_code);
1879 break;
1880 case TRB_SET_DEQ:
1881 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1882 le32_to_cpu(cmd_trb->generic.field[3])));
1883 xhci_handle_cmd_set_deq(xhci, slot_id, trb: cmd_trb, cmd_comp_code);
1884 break;
1885 case TRB_CMD_NOOP:
1886 /* Is this an aborted command turned to NO-OP? */
1887 if (cmd->status == COMP_COMMAND_RING_STOPPED)
1888 cmd_comp_code = COMP_COMMAND_RING_STOPPED;
1889 break;
1890 case TRB_RESET_EP:
1891 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1892 le32_to_cpu(cmd_trb->generic.field[3])));
1893 xhci_handle_cmd_reset_ep(xhci, slot_id, trb: cmd_trb, cmd_comp_code);
1894 break;
1895 case TRB_RESET_DEV:
1896 /* SLOT_ID field in reset device cmd completion event TRB is 0.
1897 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11)
1898 */
1899 slot_id = TRB_TO_SLOT_ID(
1900 le32_to_cpu(cmd_trb->generic.field[3]));
1901 xhci_handle_cmd_reset_dev(xhci, slot_id);
1902 break;
1903 case TRB_NEC_GET_FW:
1904 xhci_handle_cmd_nec_get_fw(xhci, event);
1905 break;
1906 case TRB_GET_BW:
1907 break;
1908 default:
1909 /* Skip over unknown commands on the event ring */
1910 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
1911 break;
1912 }
1913
1914 /* restart timer if this wasn't the last command */
1915 if (!list_is_singular(head: &xhci->cmd_list)) {
1916 xhci->current_cmd = list_first_entry(&cmd->cmd_list,
1917 struct xhci_command, cmd_list);
1918 xhci_mod_cmd_timer(xhci);
1919 } else if (xhci->current_cmd == cmd) {
1920 xhci->current_cmd = NULL;
1921 }
1922
1923event_handled:
1924 xhci_complete_del_and_free_cmd(cmd, comp_code: cmd_comp_code, COMP_PARAM(status));
1925
1926 inc_deq(xhci, ring: xhci->cmd_ring);
1927}
1928
1929static void handle_vendor_event(struct xhci_hcd *xhci,
1930 union xhci_trb *event, u32 trb_type)
1931{
1932 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1933 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1934 handle_cmd_completion(xhci, event: &event->event_cmd);
1935}
1936
1937static void handle_device_notification(struct xhci_hcd *xhci,
1938 union xhci_trb *event)
1939{
1940 u32 slot_id;
1941 struct usb_device *udev;
1942
1943 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3]));
1944 if (!xhci->devs[slot_id]) {
1945 xhci_warn(xhci, "Device Notification event for "
1946 "unused slot %u\n", slot_id);
1947 return;
1948 }
1949
1950 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
1951 slot_id);
1952 udev = xhci->devs[slot_id]->udev;
1953 if (udev && udev->parent)
1954 usb_wakeup_notification(hdev: udev->parent, portnum: udev->portnum);
1955}
1956
1957/*
1958 * Quirk hanlder for errata seen on Cavium ThunderX2 processor XHCI
1959 * Controller.
1960 * As per ThunderX2errata-129 USB 2 device may come up as USB 1
1961 * If a connection to a USB 1 device is followed by another connection
1962 * to a USB 2 device.
1963 *
1964 * Reset the PHY after the USB device is disconnected if device speed
1965 * is less than HCD_USB3.
1966 * Retry the reset sequence max of 4 times checking the PLL lock status.
1967 *
1968 */
1969static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci)
1970{
1971 struct usb_hcd *hcd = xhci_to_hcd(xhci);
1972 u32 pll_lock_check;
1973 u32 retry_count = 4;
1974
1975 do {
1976 /* Assert PHY reset */
1977 writel(val: 0x6F, addr: hcd->regs + 0x1048);
1978 udelay(usec: 10);
1979 /* De-assert the PHY reset */
1980 writel(val: 0x7F, addr: hcd->regs + 0x1048);
1981 udelay(usec: 200);
1982 pll_lock_check = readl(addr: hcd->regs + 0x1070);
1983 } while (!(pll_lock_check & 0x1) && --retry_count);
1984}
1985
1986static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event)
1987{
1988 struct usb_hcd *hcd;
1989 u32 port_id;
1990 u32 portsc, cmd_reg;
1991 int max_ports;
1992 unsigned int hcd_portnum;
1993 struct xhci_bus_state *bus_state;
1994 bool bogus_port_status = false;
1995 struct xhci_port *port;
1996
1997 /* Port status change events always have a successful completion code */
1998 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
1999 xhci_warn(xhci,
2000 "WARN: xHC returned failed port status event\n");
2001
2002 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
2003 max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
2004
2005 if ((port_id <= 0) || (port_id > max_ports)) {
2006 xhci_warn(xhci, "Port change event with invalid port ID %d\n",
2007 port_id);
2008 return;
2009 }
2010
2011 port = &xhci->hw_ports[port_id - 1];
2012 if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) {
2013 xhci_warn(xhci, "Port change event, no port for port ID %u\n",
2014 port_id);
2015 bogus_port_status = true;
2016 goto cleanup;
2017 }
2018
2019 /* We might get interrupts after shared_hcd is removed */
2020 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) {
2021 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n");
2022 bogus_port_status = true;
2023 goto cleanup;
2024 }
2025
2026 hcd = port->rhub->hcd;
2027 bus_state = &port->rhub->bus_state;
2028 hcd_portnum = port->hcd_portnum;
2029 portsc = readl(addr: port->addr);
2030
2031 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n",
2032 hcd->self.busnum, hcd_portnum + 1, port_id, portsc);
2033
2034 trace_xhci_handle_port_status(port, portsc);
2035
2036 if (hcd->state == HC_STATE_SUSPENDED) {
2037 xhci_dbg(xhci, "resume root hub\n");
2038 usb_hcd_resume_root_hub(hcd);
2039 }
2040
2041 if (hcd->speed >= HCD_USB3 &&
2042 (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) {
2043 if (port->slot_id && xhci->devs[port->slot_id])
2044 xhci->devs[port->slot_id]->flags |= VDEV_PORT_ERROR;
2045 }
2046
2047 if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) {
2048 xhci_dbg(xhci, "port resume event for port %d\n", port_id);
2049
2050 cmd_reg = readl(addr: &xhci->op_regs->command);
2051 if (!(cmd_reg & CMD_RUN)) {
2052 xhci_warn(xhci, "xHC is not running.\n");
2053 goto cleanup;
2054 }
2055
2056 if (DEV_SUPERSPEED_ANY(portsc)) {
2057 xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
2058 /* Set a flag to say the port signaled remote wakeup,
2059 * so we can tell the difference between the end of
2060 * device and host initiated resume.
2061 */
2062 bus_state->port_remote_wakeup |= 1 << hcd_portnum;
2063 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
2064 usb_hcd_start_port_resume(bus: &hcd->self, portnum: hcd_portnum);
2065 xhci_set_link_state(xhci, port, XDEV_U0);
2066 /* Need to wait until the next link state change
2067 * indicates the device is actually in U0.
2068 */
2069 bogus_port_status = true;
2070 goto cleanup;
2071 } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) {
2072 xhci_dbg(xhci, "resume HS port %d\n", port_id);
2073 port->resume_timestamp = jiffies +
2074 msecs_to_jiffies(USB_RESUME_TIMEOUT);
2075 set_bit(nr: hcd_portnum, addr: &bus_state->resuming_ports);
2076 /* Do the rest in GetPortStatus after resume time delay.
2077 * Avoid polling roothub status before that so that a
2078 * usb device auto-resume latency around ~40ms.
2079 */
2080 set_bit(HCD_FLAG_POLL_RH, addr: &hcd->flags);
2081 mod_timer(timer: &hcd->rh_timer,
2082 expires: port->resume_timestamp);
2083 usb_hcd_start_port_resume(bus: &hcd->self, portnum: hcd_portnum);
2084 bogus_port_status = true;
2085 }
2086 }
2087
2088 if ((portsc & PORT_PLC) &&
2089 DEV_SUPERSPEED_ANY(portsc) &&
2090 ((portsc & PORT_PLS_MASK) == XDEV_U0 ||
2091 (portsc & PORT_PLS_MASK) == XDEV_U1 ||
2092 (portsc & PORT_PLS_MASK) == XDEV_U2)) {
2093 xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
2094 complete(&port->u3exit_done);
2095 /* We've just brought the device into U0/1/2 through either the
2096 * Resume state after a device remote wakeup, or through the
2097 * U3Exit state after a host-initiated resume. If it's a device
2098 * initiated remote wake, don't pass up the link state change,
2099 * so the roothub behavior is consistent with external
2100 * USB 3.0 hub behavior.
2101 */
2102 if (port->slot_id && xhci->devs[port->slot_id])
2103 xhci_ring_device(xhci, slot_id: port->slot_id);
2104 if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) {
2105 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
2106 usb_wakeup_notification(hdev: hcd->self.root_hub,
2107 portnum: hcd_portnum + 1);
2108 bogus_port_status = true;
2109 goto cleanup;
2110 }
2111 }
2112
2113 /*
2114 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or
2115 * RExit to a disconnect state). If so, let the driver know it's
2116 * out of the RExit state.
2117 */
2118 if (hcd->speed < HCD_USB3 && port->rexit_active) {
2119 complete(&port->rexit_done);
2120 port->rexit_active = false;
2121 bogus_port_status = true;
2122 goto cleanup;
2123 }
2124
2125 if (hcd->speed < HCD_USB3) {
2126 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
2127 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) &&
2128 (portsc & PORT_CSC) && !(portsc & PORT_CONNECT))
2129 xhci_cavium_reset_phy_quirk(xhci);
2130 }
2131
2132cleanup:
2133
2134 /* Don't make the USB core poll the roothub if we got a bad port status
2135 * change event. Besides, at that point we can't tell which roothub
2136 * (USB 2.0 or USB 3.0) to kick.
2137 */
2138 if (bogus_port_status)
2139 return;
2140
2141 /*
2142 * xHCI port-status-change events occur when the "or" of all the
2143 * status-change bits in the portsc register changes from 0 to 1.
2144 * New status changes won't cause an event if any other change
2145 * bits are still set. When an event occurs, switch over to
2146 * polling to avoid losing status changes.
2147 */
2148 xhci_dbg(xhci, "%s: starting usb%d port polling.\n",
2149 __func__, hcd->self.busnum);
2150 set_bit(HCD_FLAG_POLL_RH, addr: &hcd->flags);
2151 spin_unlock(lock: &xhci->lock);
2152 /* Pass this up to the core */
2153 usb_hcd_poll_rh_status(hcd);
2154 spin_lock(lock: &xhci->lock);
2155}
2156
2157static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td,
2158 struct xhci_virt_ep *ep)
2159{
2160 /*
2161 * As part of low/full-speed endpoint-halt processing
2162 * we must clear the TT buffer (USB 2.0 specification 11.17.5).
2163 */
2164 if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) &&
2165 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) &&
2166 !(ep->ep_state & EP_CLEARING_TT)) {
2167 ep->ep_state |= EP_CLEARING_TT;
2168 td->urb->ep->hcpriv = td->urb->dev;
2169 if (usb_hub_clear_tt_buffer(urb: td->urb))
2170 ep->ep_state &= ~EP_CLEARING_TT;
2171 }
2172}
2173
2174/*
2175 * Check if xhci internal endpoint state has gone to a "halt" state due to an
2176 * error or stall, including default control pipe protocol stall.
2177 * The internal halt needs to be cleared with a reset endpoint command.
2178 *
2179 * External device side is also halted in functional stall cases. Class driver
2180 * will clear the device halt with a CLEAR_FEATURE(ENDPOINT_HALT) request later.
2181 */
2182static bool xhci_halted_host_endpoint(struct xhci_ep_ctx *ep_ctx, unsigned int comp_code)
2183{
2184 /* Stall halts both internal and device side endpoint */
2185 if (comp_code == COMP_STALL_ERROR)
2186 return true;
2187
2188 /* TRB completion codes that may require internal halt cleanup */
2189 if (comp_code == COMP_USB_TRANSACTION_ERROR ||
2190 comp_code == COMP_BABBLE_DETECTED_ERROR ||
2191 comp_code == COMP_SPLIT_TRANSACTION_ERROR)
2192 /*
2193 * The 0.95 spec says a babbling control endpoint is not halted.
2194 * The 0.96 spec says it is. Some HW claims to be 0.95
2195 * compliant, but it halts the control endpoint anyway.
2196 * Check endpoint context if endpoint is halted.
2197 */
2198 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED)
2199 return true;
2200
2201 return false;
2202}
2203
2204int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
2205{
2206 if (trb_comp_code >= 224 && trb_comp_code <= 255) {
2207 /* Vendor defined "informational" completion code,
2208 * treat as not-an-error.
2209 */
2210 xhci_dbg(xhci, "Vendor defined info completion code %u\n",
2211 trb_comp_code);
2212 xhci_dbg(xhci, "Treating code as success.\n");
2213 return 1;
2214 }
2215 return 0;
2216}
2217
2218static void finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2219 struct xhci_ring *ep_ring, struct xhci_td *td,
2220 u32 trb_comp_code)
2221{
2222 struct xhci_ep_ctx *ep_ctx;
2223
2224 ep_ctx = xhci_get_ep_ctx(xhci, ctx: ep->vdev->out_ctx, ep_index: ep->ep_index);
2225
2226 switch (trb_comp_code) {
2227 case COMP_STOPPED_LENGTH_INVALID:
2228 case COMP_STOPPED_SHORT_PACKET:
2229 case COMP_STOPPED:
2230 /*
2231 * The "Stop Endpoint" completion will take care of any
2232 * stopped TDs. A stopped TD may be restarted, so don't update
2233 * the ring dequeue pointer or take this TD off any lists yet.
2234 */
2235 return;
2236 case COMP_USB_TRANSACTION_ERROR:
2237 case COMP_BABBLE_DETECTED_ERROR:
2238 case COMP_SPLIT_TRANSACTION_ERROR:
2239 /*
2240 * If endpoint context state is not halted we might be
2241 * racing with a reset endpoint command issued by a unsuccessful
2242 * stop endpoint completion (context error). In that case the
2243 * td should be on the cancelled list, and EP_HALTED flag set.
2244 *
2245 * Or then it's not halted due to the 0.95 spec stating that a
2246 * babbling control endpoint should not halt. The 0.96 spec
2247 * again says it should. Some HW claims to be 0.95 compliant,
2248 * but it halts the control endpoint anyway.
2249 */
2250 if (GET_EP_CTX_STATE(ep_ctx) != EP_STATE_HALTED) {
2251 /*
2252 * If EP_HALTED is set and TD is on the cancelled list
2253 * the TD and dequeue pointer will be handled by reset
2254 * ep command completion
2255 */
2256 if ((ep->ep_state & EP_HALTED) &&
2257 !list_empty(head: &td->cancelled_td_list)) {
2258 xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n",
2259 (unsigned long long)xhci_trb_virt_to_dma(
2260 td->start_seg, td->start_trb));
2261 return;
2262 }
2263 /* endpoint not halted, don't reset it */
2264 break;
2265 }
2266 /* Almost same procedure as for STALL_ERROR below */
2267 xhci_clear_hub_tt_buffer(xhci, td, ep);
2268 xhci_handle_halted_endpoint(xhci, ep, td, reset_type: EP_HARD_RESET);
2269 return;
2270 case COMP_STALL_ERROR:
2271 /*
2272 * xhci internal endpoint state will go to a "halt" state for
2273 * any stall, including default control pipe protocol stall.
2274 * To clear the host side halt we need to issue a reset endpoint
2275 * command, followed by a set dequeue command to move past the
2276 * TD.
2277 * Class drivers clear the device side halt from a functional
2278 * stall later. Hub TT buffer should only be cleared for FS/LS
2279 * devices behind HS hubs for functional stalls.
2280 */
2281 if (ep->ep_index != 0)
2282 xhci_clear_hub_tt_buffer(xhci, td, ep);
2283
2284 xhci_handle_halted_endpoint(xhci, ep, td, reset_type: EP_HARD_RESET);
2285
2286 return; /* xhci_handle_halted_endpoint marked td cancelled */
2287 default:
2288 break;
2289 }
2290
2291 xhci_dequeue_td(xhci, td, ring: ep_ring, status: td->status);
2292}
2293
2294/* sum trb lengths from the first trb up to stop_trb, _excluding_ stop_trb */
2295static u32 sum_trb_lengths(struct xhci_td *td, union xhci_trb *stop_trb)
2296{
2297 u32 sum;
2298 union xhci_trb *trb = td->start_trb;
2299 struct xhci_segment *seg = td->start_seg;
2300
2301 for (sum = 0; trb != stop_trb; next_trb(seg: &seg, trb: &trb)) {
2302 if (!trb_is_noop(trb) && !trb_is_link(trb))
2303 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
2304 }
2305 return sum;
2306}
2307
2308/*
2309 * Process control tds, update urb status and actual_length.
2310 */
2311static void process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2312 struct xhci_ring *ep_ring, struct xhci_td *td,
2313 union xhci_trb *ep_trb, struct xhci_transfer_event *event)
2314{
2315 struct xhci_ep_ctx *ep_ctx;
2316 u32 trb_comp_code;
2317 u32 remaining, requested;
2318 u32 trb_type;
2319
2320 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
2321 ep_ctx = xhci_get_ep_ctx(xhci, ctx: ep->vdev->out_ctx, ep_index: ep->ep_index);
2322 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2323 requested = td->urb->transfer_buffer_length;
2324 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2325
2326 switch (trb_comp_code) {
2327 case COMP_SUCCESS:
2328 if (trb_type != TRB_STATUS) {
2329 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
2330 (trb_type == TRB_DATA) ? "data" : "setup");
2331 td->status = -ESHUTDOWN;
2332 break;
2333 }
2334 td->status = 0;
2335 break;
2336 case COMP_SHORT_PACKET:
2337 td->status = 0;
2338 break;
2339 case COMP_STOPPED_SHORT_PACKET:
2340 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
2341 td->urb->actual_length = remaining;
2342 else
2343 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
2344 goto finish_td;
2345 case COMP_STOPPED:
2346 switch (trb_type) {
2347 case TRB_SETUP:
2348 td->urb->actual_length = 0;
2349 goto finish_td;
2350 case TRB_DATA:
2351 case TRB_NORMAL:
2352 td->urb->actual_length = requested - remaining;
2353 goto finish_td;
2354 case TRB_STATUS:
2355 td->urb->actual_length = requested;
2356 goto finish_td;
2357 default:
2358 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n",
2359 trb_type);
2360 goto finish_td;
2361 }
2362 case COMP_STOPPED_LENGTH_INVALID:
2363 goto finish_td;
2364 default:
2365 if (!xhci_halted_host_endpoint(ep_ctx, comp_code: trb_comp_code))
2366 break;
2367 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2368 trb_comp_code, ep->ep_index);
2369 fallthrough;
2370 case COMP_STALL_ERROR:
2371 /* Did we transfer part of the data (middle) phase? */
2372 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
2373 td->urb->actual_length = requested - remaining;
2374 else if (!td->urb_length_set)
2375 td->urb->actual_length = 0;
2376 goto finish_td;
2377 }
2378
2379 /* stopped at setup stage, no data transferred */
2380 if (trb_type == TRB_SETUP)
2381 goto finish_td;
2382
2383 /*
2384 * if on data stage then update the actual_length of the URB and flag it
2385 * as set, so it won't be overwritten in the event for the last TRB.
2386 */
2387 if (trb_type == TRB_DATA ||
2388 trb_type == TRB_NORMAL) {
2389 td->urb_length_set = true;
2390 td->urb->actual_length = requested - remaining;
2391 xhci_dbg(xhci, "Waiting for status stage event\n");
2392 return;
2393 }
2394
2395 /* at status stage */
2396 if (!td->urb_length_set)
2397 td->urb->actual_length = requested;
2398
2399finish_td:
2400 finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2401}
2402
2403/*
2404 * Process isochronous tds, update urb packet status and actual_length.
2405 */
2406static void process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2407 struct xhci_ring *ep_ring, struct xhci_td *td,
2408 union xhci_trb *ep_trb, struct xhci_transfer_event *event)
2409{
2410 struct urb_priv *urb_priv;
2411 int idx;
2412 struct usb_iso_packet_descriptor *frame;
2413 u32 trb_comp_code;
2414 bool sum_trbs_for_length = false;
2415 u32 remaining, requested, ep_trb_len;
2416 int short_framestatus;
2417
2418 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2419 urb_priv = td->urb->hcpriv;
2420 idx = urb_priv->num_tds_done;
2421 frame = &td->urb->iso_frame_desc[idx];
2422 requested = frame->length;
2423 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2424 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2425 short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
2426 -EREMOTEIO : 0;
2427
2428 /* handle completion code */
2429 switch (trb_comp_code) {
2430 case COMP_SUCCESS:
2431 /* Don't overwrite status if TD had an error, see xHCI 4.9.1 */
2432 if (td->error_mid_td)
2433 break;
2434 if (remaining) {
2435 frame->status = short_framestatus;
2436 sum_trbs_for_length = true;
2437 break;
2438 }
2439 frame->status = 0;
2440 break;
2441 case COMP_SHORT_PACKET:
2442 frame->status = short_framestatus;
2443 sum_trbs_for_length = true;
2444 break;
2445 case COMP_BANDWIDTH_OVERRUN_ERROR:
2446 frame->status = -ECOMM;
2447 break;
2448 case COMP_BABBLE_DETECTED_ERROR:
2449 sum_trbs_for_length = true;
2450 fallthrough;
2451 case COMP_ISOCH_BUFFER_OVERRUN:
2452 frame->status = -EOVERFLOW;
2453 if (ep_trb != td->end_trb)
2454 td->error_mid_td = true;
2455 break;
2456 case COMP_MISSED_SERVICE_ERROR:
2457 frame->status = -EXDEV;
2458 sum_trbs_for_length = true;
2459 if (ep_trb != td->end_trb)
2460 td->error_mid_td = true;
2461 break;
2462 case COMP_INCOMPATIBLE_DEVICE_ERROR:
2463 case COMP_STALL_ERROR:
2464 frame->status = -EPROTO;
2465 break;
2466 case COMP_USB_TRANSACTION_ERROR:
2467 frame->status = -EPROTO;
2468 sum_trbs_for_length = true;
2469 if (ep_trb != td->end_trb)
2470 td->error_mid_td = true;
2471 break;
2472 case COMP_STOPPED:
2473 sum_trbs_for_length = true;
2474 break;
2475 case COMP_STOPPED_SHORT_PACKET:
2476 /* field normally containing residue now contains transferred */
2477 frame->status = short_framestatus;
2478 requested = remaining;
2479 break;
2480 case COMP_STOPPED_LENGTH_INVALID:
2481 /* exclude stopped trb with invalid length from length sum */
2482 sum_trbs_for_length = true;
2483 ep_trb_len = 0;
2484 remaining = 0;
2485 break;
2486 default:
2487 sum_trbs_for_length = true;
2488 frame->status = -1;
2489 break;
2490 }
2491
2492 if (td->urb_length_set)
2493 goto finish_td;
2494
2495 if (sum_trbs_for_length)
2496 frame->actual_length = sum_trb_lengths(td, stop_trb: ep_trb) +
2497 ep_trb_len - remaining;
2498 else
2499 frame->actual_length = requested;
2500
2501 td->urb->actual_length += frame->actual_length;
2502
2503finish_td:
2504 /* Don't give back TD yet if we encountered an error mid TD */
2505 if (td->error_mid_td && ep_trb != td->end_trb) {
2506 xhci_dbg(xhci, "Error mid isoc TD, wait for final completion event\n");
2507 td->urb_length_set = true;
2508 return;
2509 }
2510 finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2511}
2512
2513static void skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2514 struct xhci_virt_ep *ep, int status)
2515{
2516 struct urb_priv *urb_priv;
2517 struct usb_iso_packet_descriptor *frame;
2518 int idx;
2519
2520 urb_priv = td->urb->hcpriv;
2521 idx = urb_priv->num_tds_done;
2522 frame = &td->urb->iso_frame_desc[idx];
2523
2524 /* The transfer is partly done. */
2525 frame->status = -EXDEV;
2526
2527 /* calc actual length */
2528 frame->actual_length = 0;
2529
2530 xhci_dequeue_td(xhci, td, ring: ep->ring, status);
2531}
2532
2533/*
2534 * Process bulk and interrupt tds, update urb status and actual_length.
2535 */
2536static void process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2537 struct xhci_ring *ep_ring, struct xhci_td *td,
2538 union xhci_trb *ep_trb, struct xhci_transfer_event *event)
2539{
2540 struct xhci_slot_ctx *slot_ctx;
2541 u32 trb_comp_code;
2542 u32 remaining, requested, ep_trb_len;
2543
2544 slot_ctx = xhci_get_slot_ctx(xhci, ctx: ep->vdev->out_ctx);
2545 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2546 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2547 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2548 requested = td->urb->transfer_buffer_length;
2549
2550 switch (trb_comp_code) {
2551 case COMP_SUCCESS:
2552 ep->err_count = 0;
2553 /* handle success with untransferred data as short packet */
2554 if (ep_trb != td->end_trb || remaining) {
2555 xhci_warn(xhci, "WARN Successful completion on short TX\n");
2556 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2557 td->urb->ep->desc.bEndpointAddress,
2558 requested, remaining);
2559 }
2560 td->status = 0;
2561 break;
2562 case COMP_SHORT_PACKET:
2563 td->status = 0;
2564 break;
2565 case COMP_STOPPED_SHORT_PACKET:
2566 td->urb->actual_length = remaining;
2567 goto finish_td;
2568 case COMP_STOPPED_LENGTH_INVALID:
2569 /* stopped on ep trb with invalid length, exclude it */
2570 td->urb->actual_length = sum_trb_lengths(td, stop_trb: ep_trb);
2571 goto finish_td;
2572 case COMP_USB_TRANSACTION_ERROR:
2573 if (xhci->quirks & XHCI_NO_SOFT_RETRY ||
2574 (ep->err_count++ > MAX_SOFT_RETRY) ||
2575 le32_to_cpu(slot_ctx->tt_info) & TT_SLOT)
2576 break;
2577
2578 td->status = 0;
2579
2580 xhci_handle_halted_endpoint(xhci, ep, td, reset_type: EP_SOFT_RESET);
2581 return;
2582 default:
2583 /* do nothing */
2584 break;
2585 }
2586
2587 if (ep_trb == td->end_trb)
2588 td->urb->actual_length = requested - remaining;
2589 else
2590 td->urb->actual_length =
2591 sum_trb_lengths(td, stop_trb: ep_trb) +
2592 ep_trb_len - remaining;
2593finish_td:
2594 if (remaining > requested) {
2595 xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
2596 remaining);
2597 td->urb->actual_length = 0;
2598 }
2599
2600 finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2601}
2602
2603/* Transfer events which don't point to a transfer TRB, see xhci 4.17.4 */
2604static int handle_transferless_tx_event(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2605 u32 trb_comp_code)
2606{
2607 switch (trb_comp_code) {
2608 case COMP_STALL_ERROR:
2609 case COMP_USB_TRANSACTION_ERROR:
2610 case COMP_INVALID_STREAM_TYPE_ERROR:
2611 case COMP_INVALID_STREAM_ID_ERROR:
2612 xhci_dbg(xhci, "Stream transaction error ep %u no id\n", ep->ep_index);
2613 if (ep->err_count++ > MAX_SOFT_RETRY)
2614 xhci_handle_halted_endpoint(xhci, ep, NULL, reset_type: EP_HARD_RESET);
2615 else
2616 xhci_handle_halted_endpoint(xhci, ep, NULL, reset_type: EP_SOFT_RESET);
2617 break;
2618 case COMP_RING_UNDERRUN:
2619 case COMP_RING_OVERRUN:
2620 case COMP_STOPPED_LENGTH_INVALID:
2621 break;
2622 default:
2623 xhci_err(xhci, "Transfer event %u for unknown stream ring slot %u ep %u\n",
2624 trb_comp_code, ep->vdev->slot_id, ep->ep_index);
2625 return -ENODEV;
2626 }
2627 return 0;
2628}
2629
2630static bool xhci_spurious_success_tx_event(struct xhci_hcd *xhci,
2631 struct xhci_ring *ring)
2632{
2633 switch (ring->old_trb_comp_code) {
2634 case COMP_SHORT_PACKET:
2635 return xhci->quirks & XHCI_SPURIOUS_SUCCESS;
2636 case COMP_USB_TRANSACTION_ERROR:
2637 case COMP_BABBLE_DETECTED_ERROR:
2638 case COMP_ISOCH_BUFFER_OVERRUN:
2639 return xhci->quirks & XHCI_ETRON_HOST &&
2640 ring->type == TYPE_ISOC;
2641 default:
2642 return false;
2643 }
2644}
2645
2646/*
2647 * If this function returns an error condition, it means it got a Transfer
2648 * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address.
2649 * At this point, the host controller is probably hosed and should be reset.
2650 */
2651static int handle_tx_event(struct xhci_hcd *xhci,
2652 struct xhci_interrupter *ir,
2653 struct xhci_transfer_event *event)
2654{
2655 struct xhci_virt_ep *ep;
2656 struct xhci_ring *ep_ring;
2657 unsigned int slot_id;
2658 int ep_index;
2659 struct xhci_td *td = NULL;
2660 dma_addr_t ep_trb_dma;
2661 struct xhci_segment *ep_seg;
2662 union xhci_trb *ep_trb;
2663 int status = -EINPROGRESS;
2664 struct xhci_ep_ctx *ep_ctx;
2665 u32 trb_comp_code;
2666 bool ring_xrun_event = false;
2667
2668 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2669 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2670 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2671 ep_trb_dma = le64_to_cpu(event->buffer);
2672
2673 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
2674 if (!ep) {
2675 xhci_err(xhci, "ERROR Invalid Transfer event\n");
2676 goto err_out;
2677 }
2678
2679 ep_ring = xhci_dma_to_transfer_ring(ep, address: ep_trb_dma);
2680 ep_ctx = xhci_get_ep_ctx(xhci, ctx: ep->vdev->out_ctx, ep_index);
2681
2682 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
2683 xhci_err(xhci,
2684 "ERROR Transfer event for disabled endpoint slot %u ep %u\n",
2685 slot_id, ep_index);
2686 goto err_out;
2687 }
2688
2689 if (!ep_ring)
2690 return handle_transferless_tx_event(xhci, ep, trb_comp_code);
2691
2692 /* Look for common error cases */
2693 switch (trb_comp_code) {
2694 /* Skip codes that require special handling depending on
2695 * transfer type
2696 */
2697 case COMP_SUCCESS:
2698 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) {
2699 trb_comp_code = COMP_SHORT_PACKET;
2700 xhci_dbg(xhci, "Successful completion on short TX for slot %u ep %u with last td comp code %d\n",
2701 slot_id, ep_index, ep_ring->old_trb_comp_code);
2702 }
2703 break;
2704 case COMP_SHORT_PACKET:
2705 break;
2706 /* Completion codes for endpoint stopped state */
2707 case COMP_STOPPED:
2708 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n",
2709 slot_id, ep_index);
2710 break;
2711 case COMP_STOPPED_LENGTH_INVALID:
2712 xhci_dbg(xhci,
2713 "Stopped on No-op or Link TRB for slot %u ep %u\n",
2714 slot_id, ep_index);
2715 break;
2716 case COMP_STOPPED_SHORT_PACKET:
2717 xhci_dbg(xhci,
2718 "Stopped with short packet transfer detected for slot %u ep %u\n",
2719 slot_id, ep_index);
2720 break;
2721 /* Completion codes for endpoint halted state */
2722 case COMP_STALL_ERROR:
2723 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
2724 ep_index);
2725 status = -EPIPE;
2726 break;
2727 case COMP_SPLIT_TRANSACTION_ERROR:
2728 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n",
2729 slot_id, ep_index);
2730 status = -EPROTO;
2731 break;
2732 case COMP_USB_TRANSACTION_ERROR:
2733 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
2734 slot_id, ep_index);
2735 status = -EPROTO;
2736 break;
2737 case COMP_BABBLE_DETECTED_ERROR:
2738 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n",
2739 slot_id, ep_index);
2740 status = -EOVERFLOW;
2741 break;
2742 /* Completion codes for endpoint error state */
2743 case COMP_TRB_ERROR:
2744 xhci_warn(xhci,
2745 "WARN: TRB error for slot %u ep %u on endpoint\n",
2746 slot_id, ep_index);
2747 status = -EILSEQ;
2748 break;
2749 /* completion codes not indicating endpoint state change */
2750 case COMP_DATA_BUFFER_ERROR:
2751 xhci_warn(xhci,
2752 "WARN: HC couldn't access mem fast enough for slot %u ep %u\n",
2753 slot_id, ep_index);
2754 status = -ENOSR;
2755 break;
2756 case COMP_BANDWIDTH_OVERRUN_ERROR:
2757 xhci_warn(xhci,
2758 "WARN: bandwidth overrun event for slot %u ep %u on endpoint\n",
2759 slot_id, ep_index);
2760 break;
2761 case COMP_ISOCH_BUFFER_OVERRUN:
2762 xhci_warn(xhci,
2763 "WARN: buffer overrun event for slot %u ep %u on endpoint",
2764 slot_id, ep_index);
2765 break;
2766 case COMP_RING_UNDERRUN:
2767 /*
2768 * When the Isoch ring is empty, the xHC will generate
2769 * a Ring Overrun Event for IN Isoch endpoint or Ring
2770 * Underrun Event for OUT Isoch endpoint.
2771 */
2772 xhci_dbg(xhci, "Underrun event on slot %u ep %u\n", slot_id, ep_index);
2773 ring_xrun_event = true;
2774 break;
2775 case COMP_RING_OVERRUN:
2776 xhci_dbg(xhci, "Overrun event on slot %u ep %u\n", slot_id, ep_index);
2777 ring_xrun_event = true;
2778 break;
2779 case COMP_MISSED_SERVICE_ERROR:
2780 /*
2781 * When encounter missed service error, one or more isoc tds
2782 * may be missed by xHC.
2783 * Set skip flag of the ep_ring; Complete the missed tds as
2784 * short transfer when process the ep_ring next time.
2785 */
2786 ep->skip = true;
2787 xhci_dbg(xhci,
2788 "Miss service interval error for slot %u ep %u, set skip flag%s\n",
2789 slot_id, ep_index, ep_trb_dma ? ", skip now" : "");
2790 break;
2791 case COMP_NO_PING_RESPONSE_ERROR:
2792 ep->skip = true;
2793 xhci_dbg(xhci,
2794 "No Ping response error for slot %u ep %u, Skip one Isoc TD\n",
2795 slot_id, ep_index);
2796 return 0;
2797
2798 case COMP_INCOMPATIBLE_DEVICE_ERROR:
2799 /* needs disable slot command to recover */
2800 xhci_warn(xhci,
2801 "WARN: detect an incompatible device for slot %u ep %u",
2802 slot_id, ep_index);
2803 status = -EPROTO;
2804 break;
2805 default:
2806 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
2807 status = 0;
2808 break;
2809 }
2810 xhci_warn(xhci,
2811 "ERROR Unknown event condition %u for slot %u ep %u , HC probably busted\n",
2812 trb_comp_code, slot_id, ep_index);
2813 if (ep->skip)
2814 break;
2815 return 0;
2816 }
2817
2818 /*
2819 * xhci 4.10.2 states isoc endpoints should continue
2820 * processing the next TD if there was an error mid TD.
2821 * So host like NEC don't generate an event for the last
2822 * isoc TRB even if the IOC flag is set.
2823 * xhci 4.9.1 states that if there are errors in mult-TRB
2824 * TDs xHC should generate an error for that TRB, and if xHC
2825 * proceeds to the next TD it should genete an event for
2826 * any TRB with IOC flag on the way. Other host follow this.
2827 *
2828 * We wait for the final IOC event, but if we get an event
2829 * anywhere outside this TD, just give it back already.
2830 */
2831 td = list_first_entry_or_null(&ep_ring->td_list, struct xhci_td, td_list);
2832
2833 if (td && td->error_mid_td && !trb_in_td(td, suspect_dma: ep_trb_dma)) {
2834 xhci_dbg(xhci, "Missing TD completion event after mid TD error\n");
2835 xhci_dequeue_td(xhci, td, ring: ep_ring, status: td->status);
2836 }
2837
2838 /* If the TRB pointer is NULL, missed TDs will be skipped on the next event */
2839 if (trb_comp_code == COMP_MISSED_SERVICE_ERROR && !ep_trb_dma)
2840 return 0;
2841
2842 if (list_empty(head: &ep_ring->td_list)) {
2843 /*
2844 * Don't print wanings if ring is empty due to a stopped endpoint generating an
2845 * extra completion event if the device was suspended. Or, a event for the last TRB
2846 * of a short TD we already got a short event for. The short TD is already removed
2847 * from the TD list.
2848 */
2849 if (trb_comp_code != COMP_STOPPED &&
2850 trb_comp_code != COMP_STOPPED_LENGTH_INVALID &&
2851 !ring_xrun_event &&
2852 !xhci_spurious_success_tx_event(xhci, ring: ep_ring)) {
2853 xhci_warn(xhci, "Event TRB for slot %u ep %u with no TDs queued\n",
2854 slot_id, ep_index);
2855 }
2856
2857 ep->skip = false;
2858 goto check_endpoint_halted;
2859 }
2860
2861 do {
2862 td = list_first_entry(&ep_ring->td_list, struct xhci_td,
2863 td_list);
2864
2865 /* Is this a TRB in the currently executing TD? */
2866 ep_seg = trb_in_td(td, suspect_dma: ep_trb_dma);
2867
2868 if (!ep_seg) {
2869
2870 if (ep->skip && usb_endpoint_xfer_isoc(epd: &td->urb->ep->desc)) {
2871 /* this event is unlikely to match any TD, don't skip them all */
2872 if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID)
2873 return 0;
2874
2875 skip_isoc_td(xhci, td, ep, status);
2876
2877 if (!list_empty(head: &ep_ring->td_list)) {
2878 if (ring_xrun_event) {
2879 /*
2880 * If we are here, we are on xHCI 1.0 host with no
2881 * idea how many TDs were missed or where the xrun
2882 * occurred. New TDs may have been added after the
2883 * xrun, so skip only one TD to be safe.
2884 */
2885 xhci_dbg(xhci, "Skipped one TD for slot %u ep %u",
2886 slot_id, ep_index);
2887 return 0;
2888 }
2889 continue;
2890 }
2891
2892 xhci_dbg(xhci, "All TDs skipped for slot %u ep %u. Clear skip flag.\n",
2893 slot_id, ep_index);
2894 ep->skip = false;
2895 td = NULL;
2896 goto check_endpoint_halted;
2897 }
2898
2899 /* TD was queued after xrun, maybe xrun was on a link, don't panic yet */
2900 if (ring_xrun_event)
2901 return 0;
2902
2903 /*
2904 * Skip the Force Stopped Event. The 'ep_trb' of FSE is not in the current
2905 * TD pointed by 'ep_ring->dequeue' because that the hardware dequeue
2906 * pointer still at the previous TRB of the current TD. The previous TRB
2907 * maybe a Link TD or the last TRB of the previous TD. The command
2908 * completion handle will take care the rest.
2909 */
2910 if (trb_comp_code == COMP_STOPPED ||
2911 trb_comp_code == COMP_STOPPED_LENGTH_INVALID) {
2912 return 0;
2913 }
2914
2915 /*
2916 * Some hosts give a spurious success event after a short
2917 * transfer or error on last TRB. Ignore it.
2918 */
2919 if (xhci_spurious_success_tx_event(xhci, ring: ep_ring)) {
2920 xhci_dbg(xhci, "Spurious event dma %pad, comp_code %u after %u\n",
2921 &ep_trb_dma, trb_comp_code, ep_ring->old_trb_comp_code);
2922 ep_ring->old_trb_comp_code = 0;
2923 return 0;
2924 }
2925
2926 /* HC is busted, give up! */
2927 goto debug_finding_td;
2928 }
2929
2930 if (ep->skip) {
2931 xhci_dbg(xhci,
2932 "Found td. Clear skip flag for slot %u ep %u.\n",
2933 slot_id, ep_index);
2934 ep->skip = false;
2935 }
2936
2937 /*
2938 * If ep->skip is set, it means there are missed tds on the
2939 * endpoint ring need to take care of.
2940 * Process them as short transfer until reach the td pointed by
2941 * the event.
2942 */
2943 } while (ep->skip);
2944
2945 ep_ring->old_trb_comp_code = trb_comp_code;
2946
2947 /* Get out if a TD was queued at enqueue after the xrun occurred */
2948 if (ring_xrun_event)
2949 return 0;
2950
2951 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) / sizeof(*ep_trb)];
2952 trace_xhci_handle_transfer(ring: ep_ring, trb: (struct xhci_generic_trb *) ep_trb, dma: ep_trb_dma);
2953
2954 /*
2955 * No-op TRB could trigger interrupts in a case where a URB was killed
2956 * and a STALL_ERROR happens right after the endpoint ring stopped.
2957 * Reset the halted endpoint. Otherwise, the endpoint remains stalled
2958 * indefinitely.
2959 */
2960
2961 if (trb_is_noop(trb: ep_trb))
2962 goto check_endpoint_halted;
2963
2964 td->status = status;
2965
2966 /* update the urb's actual_length and give back to the core */
2967 if (usb_endpoint_xfer_control(epd: &td->urb->ep->desc))
2968 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event);
2969 else if (usb_endpoint_xfer_isoc(epd: &td->urb->ep->desc))
2970 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event);
2971 else
2972 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event);
2973 return 0;
2974
2975check_endpoint_halted:
2976 if (xhci_halted_host_endpoint(ep_ctx, comp_code: trb_comp_code))
2977 xhci_handle_halted_endpoint(xhci, ep, td, reset_type: EP_HARD_RESET);
2978
2979 return 0;
2980
2981debug_finding_td:
2982 xhci_err(xhci, "Event dma %pad for ep %d status %d not part of TD at %016llx - %016llx\n",
2983 &ep_trb_dma, ep_index, trb_comp_code,
2984 (unsigned long long)xhci_trb_virt_to_dma(td->start_seg, td->start_trb),
2985 (unsigned long long)xhci_trb_virt_to_dma(td->end_seg, td->end_trb));
2986
2987 return -ESHUTDOWN;
2988
2989err_out:
2990 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2991 (unsigned long long) xhci_trb_virt_to_dma(
2992 ir->event_ring->deq_seg,
2993 ir->event_ring->dequeue),
2994 lower_32_bits(le64_to_cpu(event->buffer)),
2995 upper_32_bits(le64_to_cpu(event->buffer)),
2996 le32_to_cpu(event->transfer_len),
2997 le32_to_cpu(event->flags));
2998 return -ENODEV;
2999}
3000
3001/*
3002 * This function handles one OS-owned event on the event ring. It may drop
3003 * xhci->lock between event processing (e.g. to pass up port status changes).
3004 */
3005static int xhci_handle_event_trb(struct xhci_hcd *xhci, struct xhci_interrupter *ir,
3006 union xhci_trb *event)
3007{
3008 u32 trb_type;
3009
3010 trace_xhci_handle_event(ring: ir->event_ring, trb: &event->generic,
3011 dma: xhci_trb_virt_to_dma(seg: ir->event_ring->deq_seg,
3012 trb: ir->event_ring->dequeue));
3013
3014 /*
3015 * Barrier between reading the TRB_CYCLE (valid) flag before, and any
3016 * speculative reads of the event's flags/data below.
3017 */
3018 rmb();
3019 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags));
3020 /* FIXME: Handle more event types. */
3021
3022 switch (trb_type) {
3023 case TRB_COMPLETION:
3024 handle_cmd_completion(xhci, event: &event->event_cmd);
3025 break;
3026 case TRB_PORT_STATUS:
3027 handle_port_status(xhci, event);
3028 break;
3029 case TRB_TRANSFER:
3030 handle_tx_event(xhci, ir, event: &event->trans_event);
3031 break;
3032 case TRB_DEV_NOTE:
3033 handle_device_notification(xhci, event);
3034 break;
3035 default:
3036 if (trb_type >= TRB_VENDOR_DEFINED_LOW)
3037 handle_vendor_event(xhci, event, trb_type);
3038 else
3039 xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type);
3040 }
3041 /* Any of the above functions may drop and re-acquire the lock, so check
3042 * to make sure a watchdog timer didn't mark the host as non-responsive.
3043 */
3044 if (xhci->xhc_state & XHCI_STATE_DYING) {
3045 xhci_dbg(xhci, "xHCI host dying, returning from event handler.\n");
3046 return -ENODEV;
3047 }
3048
3049 return 0;
3050}
3051
3052/*
3053 * Update Event Ring Dequeue Pointer:
3054 * - When all events have finished
3055 * - To avoid "Event Ring Full Error" condition
3056 */
3057void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
3058 struct xhci_interrupter *ir,
3059 bool clear_ehb)
3060{
3061 u64 temp_64;
3062 dma_addr_t deq;
3063
3064 temp_64 = xhci_read_64(xhci, regs: &ir->ir_set->erst_dequeue);
3065 deq = xhci_trb_virt_to_dma(seg: ir->event_ring->deq_seg,
3066 trb: ir->event_ring->dequeue);
3067 if (deq == 0)
3068 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n");
3069 /*
3070 * Per 4.9.4, Software writes to the ERDP register shall always advance
3071 * the Event Ring Dequeue Pointer value.
3072 */
3073 if ((temp_64 & ERST_PTR_MASK) == (deq & ERST_PTR_MASK) && !clear_ehb)
3074 return;
3075
3076 /* Update HC event ring dequeue pointer */
3077 temp_64 = ir->event_ring->deq_seg->num & ERST_DESI_MASK;
3078 temp_64 |= deq & ERST_PTR_MASK;
3079
3080 /* Clear the event handler busy flag (RW1C) */
3081 if (clear_ehb)
3082 temp_64 |= ERST_EHB;
3083 xhci_write_64(xhci, val: temp_64, regs: &ir->ir_set->erst_dequeue);
3084}
3085
3086/* Clear the interrupt pending bit for a specific interrupter. */
3087static void xhci_clear_interrupt_pending(struct xhci_interrupter *ir)
3088{
3089 if (!ir->ip_autoclear) {
3090 u32 iman;
3091
3092 iman = readl(addr: &ir->ir_set->iman);
3093 iman |= IMAN_IP;
3094 writel(val: iman, addr: &ir->ir_set->iman);
3095
3096 /* Read operation to guarantee the write has been flushed from posted buffers */
3097 readl(addr: &ir->ir_set->iman);
3098 }
3099}
3100
3101/*
3102 * Handle all OS-owned events on an interrupter event ring. It may drop
3103 * and reaquire xhci->lock between event processing.
3104 */
3105static int xhci_handle_events(struct xhci_hcd *xhci, struct xhci_interrupter *ir,
3106 bool skip_events)
3107{
3108 int event_loop = 0;
3109 int err = 0;
3110 u64 temp;
3111
3112 xhci_clear_interrupt_pending(ir);
3113
3114 /* Event ring hasn't been allocated yet. */
3115 if (!ir->event_ring || !ir->event_ring->dequeue) {
3116 xhci_err(xhci, "ERROR interrupter event ring not ready\n");
3117 return -ENOMEM;
3118 }
3119
3120 if (xhci->xhc_state & XHCI_STATE_DYING ||
3121 xhci->xhc_state & XHCI_STATE_HALTED) {
3122 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. Shouldn't IRQs be disabled?\n");
3123
3124 /* Clear the event handler busy flag (RW1C) */
3125 temp = xhci_read_64(xhci, regs: &ir->ir_set->erst_dequeue);
3126 xhci_write_64(xhci, val: temp | ERST_EHB, regs: &ir->ir_set->erst_dequeue);
3127 return -ENODEV;
3128 }
3129
3130 /* Process all OS owned event TRBs on this event ring */
3131 while (unhandled_event_trb(ring: ir->event_ring)) {
3132 if (!skip_events)
3133 err = xhci_handle_event_trb(xhci, ir, event: ir->event_ring->dequeue);
3134
3135 /*
3136 * If half a segment of events have been handled in one go then
3137 * update ERDP, and force isoc trbs to interrupt more often
3138 */
3139 if (event_loop++ > TRBS_PER_SEGMENT / 2) {
3140 xhci_update_erst_dequeue(xhci, ir, clear_ehb: false);
3141
3142 if (ir->isoc_bei_interval > AVOID_BEI_INTERVAL_MIN)
3143 ir->isoc_bei_interval = ir->isoc_bei_interval / 2;
3144
3145 event_loop = 0;
3146 }
3147
3148 /* Update SW event ring dequeue pointer */
3149 inc_deq(xhci, ring: ir->event_ring);
3150
3151 if (err)
3152 break;
3153 }
3154
3155 xhci_update_erst_dequeue(xhci, ir, clear_ehb: true);
3156
3157 return 0;
3158}
3159
3160/*
3161 * Move the event ring dequeue pointer to skip events kept in the secondary
3162 * event ring. This is used to ensure that pending events in the ring are
3163 * acknowledged, so the xHCI HCD can properly enter suspend/resume. The
3164 * secondary ring is typically maintained by an external component.
3165 */
3166void xhci_skip_sec_intr_events(struct xhci_hcd *xhci,
3167 struct xhci_ring *ring, struct xhci_interrupter *ir)
3168{
3169 union xhci_trb *current_trb;
3170 u64 erdp_reg;
3171 dma_addr_t deq;
3172
3173 /* disable irq, ack pending interrupt and ack all pending events */
3174 xhci_disable_interrupter(xhci, ir);
3175
3176 /* last acked event trb is in erdp reg */
3177 erdp_reg = xhci_read_64(xhci, regs: &ir->ir_set->erst_dequeue);
3178 deq = (dma_addr_t)(erdp_reg & ERST_PTR_MASK);
3179 if (!deq) {
3180 xhci_err(xhci, "event ring handling not required\n");
3181 return;
3182 }
3183
3184 current_trb = ir->event_ring->dequeue;
3185 /* read cycle state of the last acked trb to find out CCS */
3186 ring->cycle_state = le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE;
3187
3188 xhci_handle_events(xhci, ir, skip_events: true);
3189}
3190
3191/*
3192 * xHCI spec says we can get an interrupt, and if the HC has an error condition,
3193 * we might get bad data out of the event ring. Section 4.10.2.7 has a list of
3194 * indicators of an event TRB error, but we check the status *first* to be safe.
3195 */
3196irqreturn_t xhci_irq(struct usb_hcd *hcd)
3197{
3198 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3199 irqreturn_t ret = IRQ_HANDLED;
3200 u32 status;
3201
3202 spin_lock(lock: &xhci->lock);
3203 /* Check if the xHC generated the interrupt, or the irq is shared */
3204 status = readl(addr: &xhci->op_regs->status);
3205 if (status == ~(u32)0) {
3206 xhci_hc_died(xhci);
3207 goto out;
3208 }
3209
3210 if (!(status & STS_EINT)) {
3211 ret = IRQ_NONE;
3212 goto out;
3213 }
3214
3215 if (status & STS_HCE) {
3216 xhci_warn(xhci, "WARNING: Host Controller Error\n");
3217 goto out;
3218 }
3219
3220 if (status & STS_FATAL) {
3221 xhci_warn(xhci, "WARNING: Host System Error\n");
3222 xhci_halt(xhci);
3223 goto out;
3224 }
3225
3226 /*
3227 * Clear the op reg interrupt status first,
3228 * so we can receive interrupts from other MSI-X interrupters.
3229 * Write 1 to clear the interrupt status.
3230 */
3231 status |= STS_EINT;
3232 writel(val: status, addr: &xhci->op_regs->status);
3233
3234 /* This is the handler of the primary interrupter */
3235 xhci_handle_events(xhci, ir: xhci->interrupters[0], skip_events: false);
3236out:
3237 spin_unlock(lock: &xhci->lock);
3238
3239 return ret;
3240}
3241
3242irqreturn_t xhci_msi_irq(int irq, void *hcd)
3243{
3244 return xhci_irq(hcd);
3245}
3246EXPORT_SYMBOL_GPL(xhci_msi_irq);
3247
3248/**** Endpoint Ring Operations ****/
3249
3250/*
3251 * Generic function for queueing a TRB on a ring.
3252 * The caller must have checked to make sure there's room on the ring.
3253 *
3254 * @more_trbs_coming: Will you enqueue more TRBs before calling
3255 * prepare_transfer()?
3256 */
3257static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
3258 bool more_trbs_coming,
3259 u32 field1, u32 field2, u32 field3, u32 field4)
3260{
3261 struct xhci_generic_trb *trb;
3262
3263 trb = &ring->enqueue->generic;
3264 trb->field[0] = cpu_to_le32(field1);
3265 trb->field[1] = cpu_to_le32(field2);
3266 trb->field[2] = cpu_to_le32(field3);
3267 /* make sure TRB is fully written before giving it to the controller */
3268 wmb();
3269 trb->field[3] = cpu_to_le32(field4);
3270
3271 trace_xhci_queue_trb(ring, trb,
3272 dma: xhci_trb_virt_to_dma(seg: ring->enq_seg, trb: ring->enqueue));
3273
3274 inc_enq(xhci, ring, more_trbs_coming);
3275}
3276
3277/*
3278 * Does various checks on the endpoint ring, and makes it ready to queue num_trbs.
3279 * expand ring if it start to be full.
3280 */
3281static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
3282 u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
3283{
3284 unsigned int new_segs = 0;
3285
3286 /* Make sure the endpoint has been added to xHC schedule */
3287 switch (ep_state) {
3288 case EP_STATE_DISABLED:
3289 /*
3290 * USB core changed config/interfaces without notifying us,
3291 * or hardware is reporting the wrong state.
3292 */
3293 xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
3294 return -ENOENT;
3295 case EP_STATE_ERROR:
3296 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
3297 /* FIXME event handling code for error needs to clear it */
3298 /* XXX not sure if this should be -ENOENT or not */
3299 return -EINVAL;
3300 case EP_STATE_HALTED:
3301 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
3302 break;
3303 case EP_STATE_STOPPED:
3304 case EP_STATE_RUNNING:
3305 break;
3306 default:
3307 xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
3308 /*
3309 * FIXME issue Configure Endpoint command to try to get the HC
3310 * back into a known state.
3311 */
3312 return -EINVAL;
3313 }
3314
3315 if (ep_ring != xhci->cmd_ring) {
3316 new_segs = xhci_ring_expansion_needed(xhci, ring: ep_ring, num_trbs);
3317 } else if (xhci_num_trbs_free(ring: ep_ring) <= num_trbs) {
3318 xhci_err(xhci, "Do not support expand command ring\n");
3319 return -ENOMEM;
3320 }
3321
3322 if (new_segs) {
3323 xhci_dbg_trace(xhci, trace: trace_xhci_dbg_ring_expansion,
3324 fmt: "ERROR no room on ep ring, try ring expansion");
3325 if (xhci_ring_expansion(xhci, ring: ep_ring, num_trbs: new_segs, flags: mem_flags)) {
3326 xhci_err(xhci, "Ring expansion failed\n");
3327 return -ENOMEM;
3328 }
3329 }
3330
3331 /* Ensure that new TRBs won't overwrite a link */
3332 if (trb_is_link(trb: ep_ring->enqueue))
3333 inc_enq_past_link(xhci, ring: ep_ring, chain: 0);
3334
3335 if (last_trb_on_seg(seg: ep_ring->enq_seg, trb: ep_ring->enqueue)) {
3336 xhci_warn(xhci, "Missing link TRB at end of ring segment\n");
3337 return -EINVAL;
3338 }
3339
3340 return 0;
3341}
3342
3343static int prepare_transfer(struct xhci_hcd *xhci,
3344 struct xhci_virt_device *xdev,
3345 unsigned int ep_index,
3346 unsigned int stream_id,
3347 unsigned int num_trbs,
3348 struct urb *urb,
3349 unsigned int td_index,
3350 gfp_t mem_flags)
3351{
3352 int ret;
3353 struct urb_priv *urb_priv;
3354 struct xhci_td *td;
3355 struct xhci_ring *ep_ring;
3356 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, ctx: xdev->out_ctx, ep_index);
3357
3358 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id: xdev->slot_id, ep_index,
3359 stream_id);
3360 if (!ep_ring) {
3361 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
3362 stream_id);
3363 return -EINVAL;
3364 }
3365
3366 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
3367 num_trbs, mem_flags);
3368 if (ret)
3369 return ret;
3370
3371 urb_priv = urb->hcpriv;
3372 td = &urb_priv->td[td_index];
3373
3374 INIT_LIST_HEAD(list: &td->td_list);
3375 INIT_LIST_HEAD(list: &td->cancelled_td_list);
3376
3377 if (td_index == 0) {
3378 ret = usb_hcd_link_urb_to_ep(hcd: bus_to_hcd(bus: urb->dev->bus), urb);
3379 if (unlikely(ret))
3380 return ret;
3381 }
3382
3383 td->urb = urb;
3384 /* Add this TD to the tail of the endpoint ring's TD list */
3385 list_add_tail(new: &td->td_list, head: &ep_ring->td_list);
3386 td->start_seg = ep_ring->enq_seg;
3387 td->start_trb = ep_ring->enqueue;
3388
3389 return 0;
3390}
3391
3392unsigned int count_trbs(u64 addr, u64 len)
3393{
3394 unsigned int num_trbs;
3395
3396 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
3397 TRB_MAX_BUFF_SIZE);
3398 if (num_trbs == 0)
3399 num_trbs++;
3400
3401 return num_trbs;
3402}
3403
3404static inline unsigned int count_trbs_needed(struct urb *urb)
3405{
3406 return count_trbs(addr: urb->transfer_dma, len: urb->transfer_buffer_length);
3407}
3408
3409static unsigned int count_sg_trbs_needed(struct urb *urb)
3410{
3411 struct scatterlist *sg;
3412 unsigned int i, len, full_len, num_trbs = 0;
3413
3414 full_len = urb->transfer_buffer_length;
3415
3416 for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) {
3417 len = sg_dma_len(sg);
3418 num_trbs += count_trbs(sg_dma_address(sg), len);
3419 len = min_t(unsigned int, len, full_len);
3420 full_len -= len;
3421 if (full_len == 0)
3422 break;
3423 }
3424
3425 return num_trbs;
3426}
3427
3428static unsigned int count_isoc_trbs_needed(struct urb *urb, int i)
3429{
3430 u64 addr, len;
3431
3432 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);
3433 len = urb->iso_frame_desc[i].length;
3434
3435 return count_trbs(addr, len);
3436}
3437
3438static void check_trb_math(struct urb *urb, int running_total)
3439{
3440 if (unlikely(running_total != urb->transfer_buffer_length))
3441 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, "
3442 "queued %#x (%d), asked for %#x (%d)\n",
3443 __func__,
3444 urb->ep->desc.bEndpointAddress,
3445 running_total, running_total,
3446 urb->transfer_buffer_length,
3447 urb->transfer_buffer_length);
3448}
3449
3450static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
3451 unsigned int ep_index, unsigned int stream_id, int start_cycle,
3452 struct xhci_generic_trb *start_trb)
3453{
3454 /*
3455 * Pass all the TRBs to the hardware at once and make sure this write
3456 * isn't reordered.
3457 */
3458 wmb();
3459 if (start_cycle)
3460 start_trb->field[3] |= cpu_to_le32(start_cycle);
3461 else
3462 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
3463 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
3464}
3465
3466static void check_interval(struct urb *urb, struct xhci_ep_ctx *ep_ctx)
3467{
3468 int xhci_interval;
3469 int ep_interval;
3470
3471 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
3472 ep_interval = urb->interval;
3473
3474 /* Convert to microframes */
3475 if (urb->dev->speed == USB_SPEED_LOW ||
3476 urb->dev->speed == USB_SPEED_FULL)
3477 ep_interval *= 8;
3478
3479 /* FIXME change this to a warning and a suggestion to use the new API
3480 * to set the polling interval (once the API is added).
3481 */
3482 if (xhci_interval != ep_interval) {
3483 dev_dbg_ratelimited(&urb->dev->dev,
3484 "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n",
3485 ep_interval, str_plural(ep_interval),
3486 xhci_interval, str_plural(xhci_interval));
3487 urb->interval = xhci_interval;
3488 /* Convert back to frames for LS/FS devices */
3489 if (urb->dev->speed == USB_SPEED_LOW ||
3490 urb->dev->speed == USB_SPEED_FULL)
3491 urb->interval /= 8;
3492 }
3493}
3494
3495/*
3496 * xHCI uses normal TRBs for both bulk and interrupt. When the interrupt
3497 * endpoint is to be serviced, the xHC will consume (at most) one TD. A TD
3498 * (comprised of sg list entries) can take several service intervals to
3499 * transmit.
3500 */
3501int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3502 struct urb *urb, int slot_id, unsigned int ep_index)
3503{
3504 struct xhci_ep_ctx *ep_ctx;
3505
3506 ep_ctx = xhci_get_ep_ctx(xhci, ctx: xhci->devs[slot_id]->out_ctx, ep_index);
3507 check_interval(urb, ep_ctx);
3508
3509 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);
3510}
3511
3512/*
3513 * For xHCI 1.0 host controllers, TD size is the number of max packet sized
3514 * packets remaining in the TD (*not* including this TRB).
3515 *
3516 * Total TD packet count = total_packet_count =
3517 * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
3518 *
3519 * Packets transferred up to and including this TRB = packets_transferred =
3520 * rounddown(total bytes transferred including this TRB / wMaxPacketSize)
3521 *
3522 * TD size = total_packet_count - packets_transferred
3523 *
3524 * For xHCI 0.96 and older, TD size field should be the remaining bytes
3525 * including this TRB, right shifted by 10
3526 *
3527 * For all hosts it must fit in bits 21:17, so it can't be bigger than 31.
3528 * This is taken care of in the TRB_TD_SIZE() macro
3529 *
3530 * The last TRB in a TD must have the TD size set to zero.
3531 */
3532static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
3533 int trb_buff_len, unsigned int td_total_len,
3534 struct urb *urb, bool more_trbs_coming)
3535{
3536 u32 maxp, total_packet_count;
3537
3538 /* MTK xHCI 0.96 contains some features from 1.0 */
3539 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST))
3540 return ((td_total_len - transferred) >> 10);
3541
3542 /* One TRB with a zero-length data packet. */
3543 if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
3544 trb_buff_len == td_total_len)
3545 return 0;
3546
3547 /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */
3548 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100))
3549 trb_buff_len = 0;
3550
3551 maxp = xhci_usb_endpoint_maxp(udev: urb->dev, host_ep: urb->ep);
3552 total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
3553
3554 /* Queueing functions don't count the current TRB into transferred */
3555 return (total_packet_count - ((transferred + trb_buff_len) / maxp));
3556}
3557
3558
3559static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
3560 u32 *trb_buff_len, struct xhci_segment *seg)
3561{
3562 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
3563 unsigned int unalign;
3564 unsigned int max_pkt;
3565 u32 new_buff_len;
3566 size_t len;
3567
3568 max_pkt = xhci_usb_endpoint_maxp(udev: urb->dev, host_ep: urb->ep);
3569 unalign = (enqd_len + *trb_buff_len) % max_pkt;
3570
3571 /* we got lucky, last normal TRB data on segment is packet aligned */
3572 if (unalign == 0)
3573 return 0;
3574
3575 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n",
3576 unalign, *trb_buff_len);
3577
3578 /* is the last nornal TRB alignable by splitting it */
3579 if (*trb_buff_len > unalign) {
3580 *trb_buff_len -= unalign;
3581 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len);
3582 return 0;
3583 }
3584
3585 /*
3586 * We want enqd_len + trb_buff_len to sum up to a number aligned to
3587 * number which is divisible by the endpoint's wMaxPacketSize. IOW:
3588 * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0.
3589 */
3590 new_buff_len = max_pkt - (enqd_len % max_pkt);
3591
3592 if (new_buff_len > (urb->transfer_buffer_length - enqd_len))
3593 new_buff_len = (urb->transfer_buffer_length - enqd_len);
3594
3595 /* create a max max_pkt sized bounce buffer pointed to by last trb */
3596 if (usb_urb_dir_out(urb)) {
3597 if (urb->num_sgs) {
3598 len = sg_pcopy_to_buffer(sgl: urb->sg, nents: urb->num_sgs,
3599 buf: seg->bounce_buf, buflen: new_buff_len, skip: enqd_len);
3600 if (len != new_buff_len)
3601 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n",
3602 len, new_buff_len);
3603 } else {
3604 memcpy(to: seg->bounce_buf, from: urb->transfer_buffer + enqd_len, len: new_buff_len);
3605 }
3606
3607 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3608 max_pkt, DMA_TO_DEVICE);
3609 } else {
3610 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3611 max_pkt, DMA_FROM_DEVICE);
3612 }
3613
3614 if (dma_mapping_error(dev, dma_addr: seg->bounce_dma)) {
3615 /* try without aligning. Some host controllers survive */
3616 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
3617 return 0;
3618 }
3619 *trb_buff_len = new_buff_len;
3620 seg->bounce_len = new_buff_len;
3621 seg->bounce_offs = enqd_len;
3622
3623 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len);
3624
3625 return 1;
3626}
3627
3628/* This is very similar to what ehci-q.c qtd_fill() does */
3629int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3630 struct urb *urb, int slot_id, unsigned int ep_index)
3631{
3632 struct xhci_ring *ring;
3633 struct urb_priv *urb_priv;
3634 struct xhci_td *td;
3635 struct xhci_generic_trb *start_trb;
3636 struct scatterlist *sg = NULL;
3637 bool more_trbs_coming = true;
3638 bool need_zero_pkt = false;
3639 bool first_trb = true;
3640 unsigned int num_trbs;
3641 unsigned int start_cycle, num_sgs = 0;
3642 unsigned int enqd_len, block_len, trb_buff_len, full_len;
3643 int sent_len, ret;
3644 u32 field, length_field, remainder;
3645 u64 addr, send_addr;
3646
3647 ring = xhci_urb_to_transfer_ring(xhci, urb);
3648 if (!ring)
3649 return -EINVAL;
3650
3651 full_len = urb->transfer_buffer_length;
3652 /* If we have scatter/gather list, we use it. */
3653 if (urb->num_sgs && !(urb->transfer_flags & URB_DMA_MAP_SINGLE)) {
3654 num_sgs = urb->num_mapped_sgs;
3655 sg = urb->sg;
3656 addr = (u64) sg_dma_address(sg);
3657 block_len = sg_dma_len(sg);
3658 num_trbs = count_sg_trbs_needed(urb);
3659 } else {
3660 num_trbs = count_trbs_needed(urb);
3661 addr = (u64) urb->transfer_dma;
3662 block_len = full_len;
3663 }
3664 ret = prepare_transfer(xhci, xdev: xhci->devs[slot_id],
3665 ep_index, stream_id: urb->stream_id,
3666 num_trbs, urb, td_index: 0, mem_flags);
3667 if (unlikely(ret < 0))
3668 return ret;
3669
3670 urb_priv = urb->hcpriv;
3671
3672 /* Deal with URB_ZERO_PACKET - need one more td/trb */
3673 if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1)
3674 need_zero_pkt = true;
3675
3676 td = &urb_priv->td[0];
3677
3678 /*
3679 * Don't give the first TRB to the hardware (by toggling the cycle bit)
3680 * until we've finished creating all the other TRBs. The ring's cycle
3681 * state may change as we enqueue the other TRBs, so save it too.
3682 */
3683 start_trb = &ring->enqueue->generic;
3684 start_cycle = ring->cycle_state;
3685 send_addr = addr;
3686
3687 /* Queue the TRBs, even if they are zero-length */
3688 for (enqd_len = 0; first_trb || enqd_len < full_len;
3689 enqd_len += trb_buff_len) {
3690 field = TRB_TYPE(TRB_NORMAL);
3691
3692 /* TRB buffer should not cross 64KB boundaries */
3693 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
3694 trb_buff_len = min_t(unsigned int, trb_buff_len, block_len);
3695
3696 if (enqd_len + trb_buff_len > full_len)
3697 trb_buff_len = full_len - enqd_len;
3698
3699 /* Don't change the cycle bit of the first TRB until later */
3700 if (first_trb) {
3701 first_trb = false;
3702 if (start_cycle == 0)
3703 field |= TRB_CYCLE;
3704 } else
3705 field |= ring->cycle_state;
3706
3707 /* Chain all the TRBs together; clear the chain bit in the last
3708 * TRB to indicate it's the last TRB in the chain.
3709 */
3710 if (enqd_len + trb_buff_len < full_len) {
3711 field |= TRB_CHAIN;
3712 if (trb_is_link(trb: ring->enqueue + 1)) {
3713 if (xhci_align_td(xhci, urb, enqd_len,
3714 trb_buff_len: &trb_buff_len,
3715 seg: ring->enq_seg)) {
3716 send_addr = ring->enq_seg->bounce_dma;
3717 /* assuming TD won't span 2 segs */
3718 td->bounce_seg = ring->enq_seg;
3719 }
3720 }
3721 }
3722 if (enqd_len + trb_buff_len >= full_len) {
3723 field &= ~TRB_CHAIN;
3724 field |= TRB_IOC;
3725 more_trbs_coming = false;
3726 td->end_trb = ring->enqueue;
3727 td->end_seg = ring->enq_seg;
3728 if (xhci_urb_suitable_for_idt(urb)) {
3729 memcpy(to: &send_addr, from: urb->transfer_buffer,
3730 len: trb_buff_len);
3731 le64_to_cpus(&send_addr);
3732 field |= TRB_IDT;
3733 }
3734 }
3735
3736 /* Only set interrupt on short packet for IN endpoints */
3737 if (usb_urb_dir_in(urb))
3738 field |= TRB_ISP;
3739
3740 /* Set the TRB length, TD size, and interrupter fields. */
3741 remainder = xhci_td_remainder(xhci, transferred: enqd_len, trb_buff_len,
3742 td_total_len: full_len, urb, more_trbs_coming);
3743
3744 length_field = TRB_LEN(trb_buff_len) |
3745 TRB_TD_SIZE(remainder) |
3746 TRB_INTR_TARGET(0);
3747
3748 queue_trb(xhci, ring, more_trbs_coming: more_trbs_coming | need_zero_pkt,
3749 lower_32_bits(send_addr),
3750 upper_32_bits(send_addr),
3751 field3: length_field,
3752 field4: field);
3753 addr += trb_buff_len;
3754 sent_len = trb_buff_len;
3755
3756 while (sg && sent_len >= block_len) {
3757 /* New sg entry */
3758 --num_sgs;
3759 sent_len -= block_len;
3760 sg = sg_next(sg);
3761 if (num_sgs != 0 && sg) {
3762 block_len = sg_dma_len(sg);
3763 addr = (u64) sg_dma_address(sg);
3764 addr += sent_len;
3765 }
3766 }
3767 block_len -= sent_len;
3768 send_addr = addr;
3769 }
3770
3771 if (need_zero_pkt) {
3772 ret = prepare_transfer(xhci, xdev: xhci->devs[slot_id],
3773 ep_index, stream_id: urb->stream_id,
3774 num_trbs: 1, urb, td_index: 1, mem_flags);
3775 urb_priv->td[1].end_trb = ring->enqueue;
3776 urb_priv->td[1].end_seg = ring->enq_seg;
3777 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC;
3778 queue_trb(xhci, ring, more_trbs_coming: 0, field1: 0, field2: 0, TRB_INTR_TARGET(0), field4: field);
3779 }
3780
3781 check_trb_math(urb, running_total: enqd_len);
3782 giveback_first_trb(xhci, slot_id, ep_index, stream_id: urb->stream_id,
3783 start_cycle, start_trb);
3784 return 0;
3785}
3786
3787/* Caller must have locked xhci->lock */
3788int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3789 struct urb *urb, int slot_id, unsigned int ep_index)
3790{
3791 struct xhci_ring *ep_ring;
3792 int num_trbs;
3793 int ret;
3794 struct usb_ctrlrequest *setup;
3795 struct xhci_generic_trb *start_trb;
3796 int start_cycle;
3797 u32 field;
3798 struct urb_priv *urb_priv;
3799 struct xhci_td *td;
3800
3801 ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
3802 if (!ep_ring)
3803 return -EINVAL;
3804
3805 /*
3806 * Need to copy setup packet into setup TRB, so we can't use the setup
3807 * DMA address.
3808 */
3809 if (!urb->setup_packet)
3810 return -EINVAL;
3811
3812 if ((xhci->quirks & XHCI_ETRON_HOST) &&
3813 urb->dev->speed >= USB_SPEED_SUPER) {
3814 /*
3815 * If next available TRB is the Link TRB in the ring segment then
3816 * enqueue a No Op TRB, this can prevent the Setup and Data Stage
3817 * TRB to be breaked by the Link TRB.
3818 */
3819 if (last_trb_on_seg(seg: ep_ring->enq_seg, trb: ep_ring->enqueue + 1)) {
3820 field = TRB_TYPE(TRB_TR_NOOP) | ep_ring->cycle_state;
3821 queue_trb(xhci, ring: ep_ring, more_trbs_coming: false, field1: 0, field2: 0,
3822 TRB_INTR_TARGET(0), field4: field);
3823 }
3824 }
3825
3826 /* 1 TRB for setup, 1 for status */
3827 num_trbs = 2;
3828 /*
3829 * Don't need to check if we need additional event data and normal TRBs,
3830 * since data in control transfers will never get bigger than 16MB
3831 * XXX: can we get a buffer that crosses 64KB boundaries?
3832 */
3833 if (urb->transfer_buffer_length > 0)
3834 num_trbs++;
3835 ret = prepare_transfer(xhci, xdev: xhci->devs[slot_id],
3836 ep_index, stream_id: urb->stream_id,
3837 num_trbs, urb, td_index: 0, mem_flags);
3838 if (ret < 0)
3839 return ret;
3840
3841 urb_priv = urb->hcpriv;
3842 td = &urb_priv->td[0];
3843
3844 /*
3845 * Don't give the first TRB to the hardware (by toggling the cycle bit)
3846 * until we've finished creating all the other TRBs. The ring's cycle
3847 * state may change as we enqueue the other TRBs, so save it too.
3848 */
3849 start_trb = &ep_ring->enqueue->generic;
3850 start_cycle = ep_ring->cycle_state;
3851
3852 /* Queue setup TRB - see section 6.4.1.2.1 */
3853 /* FIXME better way to translate setup_packet into two u32 fields? */
3854 setup = (struct usb_ctrlrequest *) urb->setup_packet;
3855 field = 0;
3856 field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
3857 if (start_cycle == 0)
3858 field |= 0x1;
3859
3860 /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */
3861 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) {
3862 if (urb->transfer_buffer_length > 0) {
3863 if (setup->bRequestType & USB_DIR_IN)
3864 field |= TRB_TX_TYPE(TRB_DATA_IN);
3865 else
3866 field |= TRB_TX_TYPE(TRB_DATA_OUT);
3867 }
3868 }
3869
3870 queue_trb(xhci, ring: ep_ring, more_trbs_coming: true,
3871 field1: setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16,
3872 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16,
3873 TRB_LEN(8) | TRB_INTR_TARGET(0),
3874 /* Immediate data in pointer */
3875 field4: field);
3876
3877 /* If there's data, queue data TRBs */
3878 /* Only set interrupt on short packet for IN endpoints */
3879 if (usb_urb_dir_in(urb))
3880 field = TRB_ISP | TRB_TYPE(TRB_DATA);
3881 else
3882 field = TRB_TYPE(TRB_DATA);
3883
3884 if (urb->transfer_buffer_length > 0) {
3885 u32 length_field, remainder;
3886 u64 addr;
3887
3888 if (xhci_urb_suitable_for_idt(urb)) {
3889 memcpy(to: &addr, from: urb->transfer_buffer,
3890 len: urb->transfer_buffer_length);
3891 le64_to_cpus(&addr);
3892 field |= TRB_IDT;
3893 } else {
3894 addr = (u64) urb->transfer_dma;
3895 }
3896
3897 remainder = xhci_td_remainder(xhci, transferred: 0,
3898 trb_buff_len: urb->transfer_buffer_length,
3899 td_total_len: urb->transfer_buffer_length,
3900 urb, more_trbs_coming: 1);
3901 length_field = TRB_LEN(urb->transfer_buffer_length) |
3902 TRB_TD_SIZE(remainder) |
3903 TRB_INTR_TARGET(0);
3904 if (setup->bRequestType & USB_DIR_IN)
3905 field |= TRB_DIR_IN;
3906 queue_trb(xhci, ring: ep_ring, more_trbs_coming: true,
3907 lower_32_bits(addr),
3908 upper_32_bits(addr),
3909 field3: length_field,
3910 field4: field | ep_ring->cycle_state);
3911 }
3912
3913 /* Save the DMA address of the last TRB in the TD */
3914 td->end_trb = ep_ring->enqueue;
3915 td->end_seg = ep_ring->enq_seg;
3916
3917 /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */
3918 /* If the device sent data, the status stage is an OUT transfer */
3919 if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN)
3920 field = 0;
3921 else
3922 field = TRB_DIR_IN;
3923 queue_trb(xhci, ring: ep_ring, more_trbs_coming: false,
3924 field1: 0,
3925 field2: 0,
3926 TRB_INTR_TARGET(0),
3927 /* Event on completion */
3928 field4: field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);
3929
3930 giveback_first_trb(xhci, slot_id, ep_index, stream_id: 0,
3931 start_cycle, start_trb);
3932 return 0;
3933}
3934
3935/*
3936 * The transfer burst count field of the isochronous TRB defines the number of
3937 * bursts that are required to move all packets in this TD. Only SuperSpeed
3938 * devices can burst up to bMaxBurst number of packets per service interval.
3939 * This field is zero based, meaning a value of zero in the field means one
3940 * burst. Basically, for everything but SuperSpeed devices, this field will be
3941 * zero. Only xHCI 1.0 host controllers support this field.
3942 */
3943static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
3944 struct urb *urb, unsigned int total_packet_count)
3945{
3946 unsigned int max_burst;
3947
3948 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER)
3949 return 0;
3950
3951 max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3952 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
3953}
3954
3955/*
3956 * Returns the number of packets in the last "burst" of packets. This field is
3957 * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so
3958 * the last burst packet count is equal to the total number of packets in the
3959 * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst
3960 * must contain (bMaxBurst + 1) number of packets, but the last burst can
3961 * contain 1 to (bMaxBurst + 1) packets.
3962 */
3963static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
3964 struct urb *urb, unsigned int total_packet_count)
3965{
3966 unsigned int max_burst;
3967 unsigned int residue;
3968
3969 if (xhci->hci_version < 0x100)
3970 return 0;
3971
3972 if (urb->dev->speed >= USB_SPEED_SUPER) {
3973 /* bMaxBurst is zero based: 0 means 1 packet per burst */
3974 max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3975 residue = total_packet_count % (max_burst + 1);
3976 /* If residue is zero, the last burst contains (max_burst + 1)
3977 * number of packets, but the TLBPC field is zero-based.
3978 */
3979 if (residue == 0)
3980 return max_burst;
3981 return residue - 1;
3982 }
3983 if (total_packet_count == 0)
3984 return 0;
3985 return total_packet_count - 1;
3986}
3987
3988/*
3989 * Calculates Frame ID field of the isochronous TRB identifies the
3990 * target frame that the Interval associated with this Isochronous
3991 * Transfer Descriptor will start on. Refer to 4.11.2.5 in 1.1 spec.
3992 *
3993 * Returns actual frame id on success, negative value on error.
3994 */
3995static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci,
3996 struct urb *urb, int index)
3997{
3998 int start_frame, ist, ret = 0;
3999 int start_frame_id, end_frame_id, current_frame_id;
4000
4001 if (urb->dev->speed == USB_SPEED_LOW ||
4002 urb->dev->speed == USB_SPEED_FULL)
4003 start_frame = urb->start_frame + index * urb->interval;
4004 else
4005 start_frame = (urb->start_frame + index * urb->interval) >> 3;
4006
4007 /* Isochronous Scheduling Threshold (IST, bits 0~3 in HCSPARAMS2):
4008 *
4009 * If bit [3] of IST is cleared to '0', software can add a TRB no
4010 * later than IST[2:0] Microframes before that TRB is scheduled to
4011 * be executed.
4012 * If bit [3] of IST is set to '1', software can add a TRB no later
4013 * than IST[2:0] Frames before that TRB is scheduled to be executed.
4014 */
4015 ist = HCS_IST(xhci->hcs_params2) & 0x7;
4016 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
4017 ist <<= 3;
4018
4019 /* Software shall not schedule an Isoch TD with a Frame ID value that
4020 * is less than the Start Frame ID or greater than the End Frame ID,
4021 * where:
4022 *
4023 * End Frame ID = (Current MFINDEX register value + 895 ms.) MOD 2048
4024 * Start Frame ID = (Current MFINDEX register value + IST + 1) MOD 2048
4025 *
4026 * Both the End Frame ID and Start Frame ID values are calculated
4027 * in microframes. When software determines the valid Frame ID value;
4028 * The End Frame ID value should be rounded down to the nearest Frame
4029 * boundary, and the Start Frame ID value should be rounded up to the
4030 * nearest Frame boundary.
4031 */
4032 current_frame_id = readl(addr: &xhci->run_regs->microframe_index);
4033 start_frame_id = roundup(current_frame_id + ist + 1, 8);
4034 end_frame_id = rounddown(current_frame_id + 895 * 8, 8);
4035
4036 start_frame &= 0x7ff;
4037 start_frame_id = (start_frame_id >> 3) & 0x7ff;
4038 end_frame_id = (end_frame_id >> 3) & 0x7ff;
4039
4040 if (start_frame_id < end_frame_id) {
4041 if (start_frame > end_frame_id ||
4042 start_frame < start_frame_id)
4043 ret = -EINVAL;
4044 } else if (start_frame_id > end_frame_id) {
4045 if ((start_frame > end_frame_id &&
4046 start_frame < start_frame_id))
4047 ret = -EINVAL;
4048 } else {
4049 ret = -EINVAL;
4050 }
4051
4052 if (index == 0) {
4053 if (ret == -EINVAL || start_frame == start_frame_id) {
4054 start_frame = start_frame_id + 1;
4055 if (urb->dev->speed == USB_SPEED_LOW ||
4056 urb->dev->speed == USB_SPEED_FULL)
4057 urb->start_frame = start_frame;
4058 else
4059 urb->start_frame = start_frame << 3;
4060 ret = 0;
4061 }
4062 }
4063
4064 if (ret) {
4065 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n",
4066 start_frame, current_frame_id, index,
4067 start_frame_id, end_frame_id);
4068 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n");
4069 return ret;
4070 }
4071
4072 return start_frame;
4073}
4074
4075/* Check if we should generate event interrupt for a TD in an isoc URB */
4076static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i,
4077 struct xhci_interrupter *ir)
4078{
4079 if (xhci->hci_version < 0x100)
4080 return false;
4081 /* always generate an event interrupt for the last TD */
4082 if (i == num_tds - 1)
4083 return false;
4084 /*
4085 * If AVOID_BEI is set the host handles full event rings poorly,
4086 * generate an event at least every 8th TD to clear the event ring
4087 */
4088 if (i && ir->isoc_bei_interval && xhci->quirks & XHCI_AVOID_BEI)
4089 return !!(i % ir->isoc_bei_interval);
4090
4091 return true;
4092}
4093
4094/* This is for isoc transfer */
4095static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
4096 struct urb *urb, int slot_id, unsigned int ep_index)
4097{
4098 struct xhci_interrupter *ir;
4099 struct xhci_ring *ep_ring;
4100 struct urb_priv *urb_priv;
4101 struct xhci_td *td;
4102 int num_tds, trbs_per_td;
4103 struct xhci_generic_trb *start_trb;
4104 bool first_trb;
4105 int start_cycle;
4106 u32 field, length_field;
4107 int running_total, trb_buff_len, td_len, td_remain_len, ret;
4108 u64 start_addr, addr;
4109 int i, j;
4110 bool more_trbs_coming;
4111 struct xhci_virt_ep *xep;
4112 int frame_id;
4113
4114 xep = &xhci->devs[slot_id]->eps[ep_index];
4115 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
4116 ir = xhci->interrupters[0];
4117
4118 num_tds = urb->number_of_packets;
4119 if (num_tds < 1) {
4120 xhci_dbg(xhci, "Isoc URB with zero packets?\n");
4121 return -EINVAL;
4122 }
4123 start_addr = (u64) urb->transfer_dma;
4124 start_trb = &ep_ring->enqueue->generic;
4125 start_cycle = ep_ring->cycle_state;
4126
4127 urb_priv = urb->hcpriv;
4128 /* Queue the TRBs for each TD, even if they are zero-length */
4129 for (i = 0; i < num_tds; i++) {
4130 unsigned int total_pkt_count, max_pkt;
4131 unsigned int burst_count, last_burst_pkt_count;
4132 u32 sia_frame_id;
4133
4134 first_trb = true;
4135 running_total = 0;
4136 addr = start_addr + urb->iso_frame_desc[i].offset;
4137 td_len = urb->iso_frame_desc[i].length;
4138 td_remain_len = td_len;
4139 max_pkt = xhci_usb_endpoint_maxp(udev: urb->dev, host_ep: urb->ep);
4140 total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
4141
4142 /* A zero-length transfer still involves at least one packet. */
4143 if (total_pkt_count == 0)
4144 total_pkt_count++;
4145 burst_count = xhci_get_burst_count(xhci, urb, total_packet_count: total_pkt_count);
4146 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci,
4147 urb, total_packet_count: total_pkt_count);
4148
4149 trbs_per_td = count_isoc_trbs_needed(urb, i);
4150
4151 ret = prepare_transfer(xhci, xdev: xhci->devs[slot_id], ep_index,
4152 stream_id: urb->stream_id, num_trbs: trbs_per_td, urb, td_index: i, mem_flags);
4153 if (ret < 0) {
4154 if (i == 0)
4155 return ret;
4156 goto cleanup;
4157 }
4158 td = &urb_priv->td[i];
4159 /* use SIA as default, if frame id is used overwrite it */
4160 sia_frame_id = TRB_SIA;
4161 if (!(urb->transfer_flags & URB_ISO_ASAP) &&
4162 HCC_CFC(xhci->hcc_params)) {
4163 frame_id = xhci_get_isoc_frame_id(xhci, urb, index: i);
4164 if (frame_id >= 0)
4165 sia_frame_id = TRB_FRAME_ID(frame_id);
4166 }
4167 /*
4168 * Set isoc specific data for the first TRB in a TD.
4169 * Prevent HW from getting the TRBs by keeping the cycle state
4170 * inverted in the first TDs isoc TRB.
4171 */
4172 field = TRB_TYPE(TRB_ISOC) |
4173 TRB_TLBPC(last_burst_pkt_count) |
4174 sia_frame_id |
4175 (i ? ep_ring->cycle_state : !start_cycle);
4176
4177 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */
4178 if (!xep->use_extended_tbc)
4179 field |= TRB_TBC(burst_count);
4180
4181 /* fill the rest of the TRB fields, and remaining normal TRBs */
4182 for (j = 0; j < trbs_per_td; j++) {
4183 u32 remainder = 0;
4184
4185 /* only first TRB is isoc, overwrite otherwise */
4186 if (!first_trb)
4187 field = TRB_TYPE(TRB_NORMAL) |
4188 ep_ring->cycle_state;
4189
4190 /* Only set interrupt on short packet for IN EPs */
4191 if (usb_urb_dir_in(urb))
4192 field |= TRB_ISP;
4193
4194 /* Set the chain bit for all except the last TRB */
4195 if (j < trbs_per_td - 1) {
4196 more_trbs_coming = true;
4197 field |= TRB_CHAIN;
4198 } else {
4199 more_trbs_coming = false;
4200 td->end_trb = ep_ring->enqueue;
4201 td->end_seg = ep_ring->enq_seg;
4202 field |= TRB_IOC;
4203 if (trb_block_event_intr(xhci, num_tds, i, ir))
4204 field |= TRB_BEI;
4205 }
4206 /* Calculate TRB length */
4207 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
4208 if (trb_buff_len > td_remain_len)
4209 trb_buff_len = td_remain_len;
4210
4211 /* Set the TRB length, TD size, & interrupter fields. */
4212 remainder = xhci_td_remainder(xhci, transferred: running_total,
4213 trb_buff_len, td_total_len: td_len,
4214 urb, more_trbs_coming);
4215
4216 length_field = TRB_LEN(trb_buff_len) |
4217 TRB_INTR_TARGET(0);
4218
4219 /* xhci 1.1 with ETE uses TD Size field for TBC */
4220 if (first_trb && xep->use_extended_tbc)
4221 length_field |= TRB_TD_SIZE_TBC(burst_count);
4222 else
4223 length_field |= TRB_TD_SIZE(remainder);
4224 first_trb = false;
4225
4226 queue_trb(xhci, ring: ep_ring, more_trbs_coming,
4227 lower_32_bits(addr),
4228 upper_32_bits(addr),
4229 field3: length_field,
4230 field4: field);
4231 running_total += trb_buff_len;
4232
4233 addr += trb_buff_len;
4234 td_remain_len -= trb_buff_len;
4235 }
4236
4237 /* Check TD length */
4238 if (running_total != td_len) {
4239 xhci_err(xhci, "ISOC TD length unmatch\n");
4240 ret = -EINVAL;
4241 goto cleanup;
4242 }
4243 }
4244
4245 /* store the next frame id */
4246 if (HCC_CFC(xhci->hcc_params))
4247 xep->next_frame_id = urb->start_frame + num_tds * urb->interval;
4248
4249 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
4250 if (xhci->quirks & XHCI_AMD_PLL_FIX)
4251 usb_amd_quirk_pll_disable();
4252 }
4253 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
4254
4255 giveback_first_trb(xhci, slot_id, ep_index, stream_id: urb->stream_id,
4256 start_cycle, start_trb);
4257 return 0;
4258cleanup:
4259 /* Clean up a partially enqueued isoc transfer. */
4260
4261 for (i--; i >= 0; i--)
4262 list_del_init(entry: &urb_priv->td[i].td_list);
4263
4264 /* Use the first TD as a temporary variable to turn the TDs we've queued
4265 * into No-ops with a software-owned cycle bit. That way the hardware
4266 * won't accidentally start executing bogus TDs when we partially
4267 * overwrite them. td->start_trb and td->start_seg are already set.
4268 */
4269 urb_priv->td[0].end_trb = ep_ring->enqueue;
4270 /* Every TRB except the first & last will have its cycle bit flipped. */
4271 td_to_noop(td: &urb_priv->td[0], flip_cycle: true);
4272
4273 /* Reset the ring enqueue back to the first TRB and its cycle bit. */
4274 ep_ring->enqueue = urb_priv->td[0].start_trb;
4275 ep_ring->enq_seg = urb_priv->td[0].start_seg;
4276 ep_ring->cycle_state = start_cycle;
4277 usb_hcd_unlink_urb_from_ep(hcd: bus_to_hcd(bus: urb->dev->bus), urb);
4278 return ret;
4279}
4280
4281/*
4282 * Check transfer ring to guarantee there is enough room for the urb.
4283 * Update ISO URB start_frame and interval.
4284 * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
4285 * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
4286 * Contiguous Frame ID is not supported by HC.
4287 */
4288int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
4289 struct urb *urb, int slot_id, unsigned int ep_index)
4290{
4291 struct xhci_virt_device *xdev;
4292 struct xhci_ring *ep_ring;
4293 struct xhci_ep_ctx *ep_ctx;
4294 int start_frame;
4295 int num_tds, num_trbs, i;
4296 int ret;
4297 struct xhci_virt_ep *xep;
4298 int ist;
4299
4300 xdev = xhci->devs[slot_id];
4301 xep = &xhci->devs[slot_id]->eps[ep_index];
4302 ep_ring = xdev->eps[ep_index].ring;
4303 ep_ctx = xhci_get_ep_ctx(xhci, ctx: xdev->out_ctx, ep_index);
4304
4305 num_trbs = 0;
4306 num_tds = urb->number_of_packets;
4307 for (i = 0; i < num_tds; i++)
4308 num_trbs += count_isoc_trbs_needed(urb, i);
4309
4310 /* Check the ring to guarantee there is enough room for the whole urb.
4311 * Do not insert any td of the urb to the ring if the check failed.
4312 */
4313 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
4314 num_trbs, mem_flags);
4315 if (ret)
4316 return ret;
4317
4318 /*
4319 * Check interval value. This should be done before we start to
4320 * calculate the start frame value.
4321 */
4322 check_interval(urb, ep_ctx);
4323
4324 /* Calculate the start frame and put it in urb->start_frame. */
4325 if (HCC_CFC(xhci->hcc_params) && !list_empty(head: &ep_ring->td_list)) {
4326 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_RUNNING) {
4327 urb->start_frame = xep->next_frame_id;
4328 goto skip_start_over;
4329 }
4330 }
4331
4332 start_frame = readl(addr: &xhci->run_regs->microframe_index);
4333 start_frame &= 0x3fff;
4334 /*
4335 * Round up to the next frame and consider the time before trb really
4336 * gets scheduled by hardare.
4337 */
4338 ist = HCS_IST(xhci->hcs_params2) & 0x7;
4339 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
4340 ist <<= 3;
4341 start_frame += ist + XHCI_CFC_DELAY;
4342 start_frame = roundup(start_frame, 8);
4343
4344 /*
4345 * Round up to the next ESIT (Endpoint Service Interval Time) if ESIT
4346 * is greate than 8 microframes.
4347 */
4348 if (urb->dev->speed == USB_SPEED_LOW ||
4349 urb->dev->speed == USB_SPEED_FULL) {
4350 start_frame = roundup(start_frame, urb->interval << 3);
4351 urb->start_frame = start_frame >> 3;
4352 } else {
4353 start_frame = roundup(start_frame, urb->interval);
4354 urb->start_frame = start_frame;
4355 }
4356
4357skip_start_over:
4358
4359 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);
4360}
4361
4362/**** Command Ring Operations ****/
4363
4364/* Generic function for queueing a command TRB on the command ring.
4365 * Check to make sure there's room on the command ring for one command TRB.
4366 * Also check that there's room reserved for commands that must not fail.
4367 * If this is a command that must not fail, meaning command_must_succeed = TRUE,
4368 * then only check for the number of reserved spots.
4369 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4370 * because the command event handler may want to resubmit a failed command.
4371 */
4372static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4373 u32 field1, u32 field2,
4374 u32 field3, u32 field4, bool command_must_succeed)
4375{
4376 int reserved_trbs = xhci->cmd_ring_reserved_trbs;
4377 int ret;
4378
4379 if ((xhci->xhc_state & XHCI_STATE_DYING) ||
4380 (xhci->xhc_state & XHCI_STATE_HALTED)) {
4381 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command. state: 0x%x\n",
4382 xhci->xhc_state);
4383 return -ESHUTDOWN;
4384 }
4385
4386 if (!command_must_succeed)
4387 reserved_trbs++;
4388
4389 ret = prepare_ring(xhci, ep_ring: xhci->cmd_ring, EP_STATE_RUNNING,
4390 num_trbs: reserved_trbs, GFP_ATOMIC);
4391 if (ret < 0) {
4392 xhci_err(xhci, "ERR: No room for command on command ring\n");
4393 if (command_must_succeed)
4394 xhci_err(xhci, "ERR: Reserved TRB counting for "
4395 "unfailable commands failed.\n");
4396 return ret;
4397 }
4398
4399 cmd->command_trb = xhci->cmd_ring->enqueue;
4400
4401 /* if there are no other commands queued we start the timeout timer */
4402 if (list_empty(head: &xhci->cmd_list)) {
4403 xhci->current_cmd = cmd;
4404 xhci_mod_cmd_timer(xhci);
4405 }
4406
4407 list_add_tail(new: &cmd->cmd_list, head: &xhci->cmd_list);
4408
4409 queue_trb(xhci, ring: xhci->cmd_ring, more_trbs_coming: false, field1, field2, field3,
4410 field4: field4 | xhci->cmd_ring->cycle_state);
4411 return 0;
4412}
4413
4414/* Queue a slot enable or disable request on the command ring */
4415int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
4416 u32 trb_type, u32 slot_id)
4417{
4418 return queue_command(xhci, cmd, field1: 0, field2: 0, field3: 0,
4419 TRB_TYPE(trb_type) | SLOT_ID_FOR_TRB(slot_id), command_must_succeed: false);
4420}
4421
4422/* Queue an address device command TRB */
4423int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4424 dma_addr_t in_ctx_ptr, u32 slot_id, enum xhci_setup_dev setup)
4425{
4426 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4427 upper_32_bits(in_ctx_ptr), field3: 0,
4428 TRB_TYPE(TRB_ADDR_DEV) | SLOT_ID_FOR_TRB(slot_id)
4429 | (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0), command_must_succeed: false);
4430}
4431
4432int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4433 u32 field1, u32 field2, u32 field3, u32 field4)
4434{
4435 return queue_command(xhci, cmd, field1, field2, field3, field4, command_must_succeed: false);
4436}
4437
4438/* Queue a reset device command TRB */
4439int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4440 u32 slot_id)
4441{
4442 return queue_command(xhci, cmd, field1: 0, field2: 0, field3: 0,
4443 TRB_TYPE(TRB_RESET_DEV) | SLOT_ID_FOR_TRB(slot_id),
4444 command_must_succeed: false);
4445}
4446
4447/* Queue a configure endpoint command TRB */
4448int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
4449 struct xhci_command *cmd, dma_addr_t in_ctx_ptr,
4450 u32 slot_id, bool command_must_succeed)
4451{
4452 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4453 upper_32_bits(in_ctx_ptr), field3: 0,
4454 TRB_TYPE(TRB_CONFIG_EP) | SLOT_ID_FOR_TRB(slot_id),
4455 command_must_succeed);
4456}
4457
4458/* Queue a get root hub port bandwidth command TRB */
4459int xhci_queue_get_port_bw(struct xhci_hcd *xhci,
4460 struct xhci_command *cmd, dma_addr_t in_ctx_ptr,
4461 u8 dev_speed, bool command_must_succeed)
4462{
4463 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4464 upper_32_bits(in_ctx_ptr), field3: 0,
4465 TRB_TYPE(TRB_GET_BW) | DEV_SPEED_FOR_TRB(dev_speed),
4466 command_must_succeed);
4467}
4468
4469/* Queue an evaluate context command TRB */
4470int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
4471 dma_addr_t in_ctx_ptr, u32 slot_id, bool command_must_succeed)
4472{
4473 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4474 upper_32_bits(in_ctx_ptr), field3: 0,
4475 TRB_TYPE(TRB_EVAL_CONTEXT) | SLOT_ID_FOR_TRB(slot_id),
4476 command_must_succeed);
4477}
4478
4479/*
4480 * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
4481 * activity on an endpoint that is about to be suspended.
4482 */
4483int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
4484 int slot_id, unsigned int ep_index, int suspend)
4485{
4486 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4487 u32 trb_ep_index = EP_INDEX_FOR_TRB(ep_index);
4488 u32 type = TRB_TYPE(TRB_STOP_RING);
4489 u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend);
4490
4491 return queue_command(xhci, cmd, field1: 0, field2: 0, field3: 0,
4492 field4: trb_slot_id | trb_ep_index | type | trb_suspend, command_must_succeed: false);
4493}
4494
4495int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
4496 int slot_id, unsigned int ep_index,
4497 enum xhci_ep_reset_type reset_type)
4498{
4499 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4500 u32 trb_ep_index = EP_INDEX_FOR_TRB(ep_index);
4501 u32 type = TRB_TYPE(TRB_RESET_EP);
4502
4503 if (reset_type == EP_SOFT_RESET)
4504 type |= TRB_TSP;
4505
4506 return queue_command(xhci, cmd, field1: 0, field2: 0, field3: 0,
4507 field4: trb_slot_id | trb_ep_index | type, command_must_succeed: false);
4508}
4509