1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Cadence USBHS-DEV Driver - gadget side.
4 *
5 * Copyright (C) 2023 Cadence Design Systems.
6 *
7 * Authors: Pawel Laszczak <pawell@cadence.com>
8 */
9
10 /*
11 * Work around 1:
12 * At some situations, the controller may get stale data address in TRB
13 * at below sequences:
14 * 1. Controller read TRB includes data address
15 * 2. Software updates TRBs includes data address and Cycle bit
16 * 3. Controller read TRB which includes Cycle bit
17 * 4. DMA run with stale data address
18 *
19 * To fix this problem, driver needs to make the first TRB in TD as invalid.
20 * After preparing all TRBs driver needs to check the position of DMA and
21 * if the DMA point to the first just added TRB and doorbell is 1,
22 * then driver must defer making this TRB as valid. This TRB will be make
23 * as valid during adding next TRB only if DMA is stopped or at TRBERR
24 * interrupt.
25 *
26 */
27
28 #include <linux/dma-mapping.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/interrupt.h>
31 #include <linux/property.h>
32 #include <linux/dmapool.h>
33 #include <linux/iopoll.h>
34
35 #include "cdns2-gadget.h"
36 #include "cdns2-trace.h"
37
38 /**
39 * set_reg_bit_32 - set bit in given 32 bits register.
40 * @ptr: register address.
41 * @mask: bits to set.
42 */
set_reg_bit_32(void __iomem * ptr,u32 mask)43 static void set_reg_bit_32(void __iomem *ptr, u32 mask)
44 {
45 mask = readl(ptr) | mask;
46 writel(mask, ptr);
47 }
48
49 /*
50 * clear_reg_bit_32 - clear bit in given 32 bits register.
51 * @ptr: register address.
52 * @mask: bits to clear.
53 */
clear_reg_bit_32(void __iomem * ptr,u32 mask)54 static void clear_reg_bit_32(void __iomem *ptr, u32 mask)
55 {
56 mask = readl(ptr) & ~mask;
57 writel(mask, ptr);
58 }
59
60 /* Clear bit in given 8 bits register. */
clear_reg_bit_8(void __iomem * ptr,u8 mask)61 static void clear_reg_bit_8(void __iomem *ptr, u8 mask)
62 {
63 mask = readb(ptr) & ~mask;
64 writeb(mask, ptr);
65 }
66
67 /* Set bit in given 16 bits register. */
set_reg_bit_8(void __iomem * ptr,u8 mask)68 void set_reg_bit_8(void __iomem *ptr, u8 mask)
69 {
70 mask = readb(ptr) | mask;
71 writeb(mask, ptr);
72 }
73
cdns2_get_dma_pos(struct cdns2_device * pdev,struct cdns2_endpoint * pep)74 static int cdns2_get_dma_pos(struct cdns2_device *pdev,
75 struct cdns2_endpoint *pep)
76 {
77 int dma_index;
78
79 dma_index = readl(&pdev->adma_regs->ep_traddr) - pep->ring.dma;
80
81 return dma_index / TRB_SIZE;
82 }
83
84 /* Get next private request from list. */
cdns2_next_preq(struct list_head * list)85 struct cdns2_request *cdns2_next_preq(struct list_head *list)
86 {
87 return list_first_entry_or_null(list, struct cdns2_request, list);
88 }
89
cdns2_select_ep(struct cdns2_device * pdev,u32 ep)90 void cdns2_select_ep(struct cdns2_device *pdev, u32 ep)
91 {
92 if (pdev->selected_ep == ep)
93 return;
94
95 pdev->selected_ep = ep;
96 writel(ep, &pdev->adma_regs->ep_sel);
97 }
98
cdns2_trb_virt_to_dma(struct cdns2_endpoint * pep,struct cdns2_trb * trb)99 dma_addr_t cdns2_trb_virt_to_dma(struct cdns2_endpoint *pep,
100 struct cdns2_trb *trb)
101 {
102 u32 offset = (char *)trb - (char *)pep->ring.trbs;
103
104 return pep->ring.dma + offset;
105 }
106
cdns2_free_tr_segment(struct cdns2_endpoint * pep)107 static void cdns2_free_tr_segment(struct cdns2_endpoint *pep)
108 {
109 struct cdns2_device *pdev = pep->pdev;
110 struct cdns2_ring *ring = &pep->ring;
111
112 if (pep->ring.trbs) {
113 dma_pool_free(pdev->eps_dma_pool, ring->trbs, ring->dma);
114 memset(ring, 0, sizeof(*ring));
115 }
116 }
117
118 /* Allocates Transfer Ring segment. */
cdns2_alloc_tr_segment(struct cdns2_endpoint * pep)119 static int cdns2_alloc_tr_segment(struct cdns2_endpoint *pep)
120 {
121 struct cdns2_device *pdev = pep->pdev;
122 struct cdns2_trb *link_trb;
123 struct cdns2_ring *ring;
124
125 ring = &pep->ring;
126
127 if (!ring->trbs) {
128 ring->trbs = dma_pool_alloc(pdev->eps_dma_pool,
129 GFP_DMA32 | GFP_ATOMIC,
130 &ring->dma);
131 if (!ring->trbs)
132 return -ENOMEM;
133 }
134
135 memset(ring->trbs, 0, TR_SEG_SIZE);
136
137 if (!pep->num)
138 return 0;
139
140 /* Initialize the last TRB as Link TRB */
141 link_trb = (ring->trbs + (TRBS_PER_SEGMENT - 1));
142 link_trb->buffer = cpu_to_le32(TRB_BUFFER(ring->dma));
143 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) |
144 TRB_TOGGLE);
145
146 return 0;
147 }
148
149 /*
150 * Stalls and flushes selected endpoint.
151 * Endpoint must be selected before invoking this function.
152 */
cdns2_ep_stall_flush(struct cdns2_endpoint * pep)153 static void cdns2_ep_stall_flush(struct cdns2_endpoint *pep)
154 {
155 struct cdns2_device *pdev = pep->pdev;
156 int val;
157
158 trace_cdns2_ep_halt(pep, 1, 1);
159
160 writel(DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
161
162 /* Wait for DFLUSH cleared. */
163 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
164 !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
165 pep->ep_state |= EP_STALLED;
166 pep->ep_state &= ~EP_STALL_PENDING;
167 }
168
169 /*
170 * Increment a trb index.
171 *
172 * The index should never point to the last link TRB in TR. After incrementing,
173 * if it point to the link TRB, wrap around to the beginning and revert
174 * cycle state bit. The link TRB is always at the last TRB entry.
175 */
cdns2_ep_inc_trb(int * index,u8 * cs,int trb_in_seg)176 static void cdns2_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
177 {
178 (*index)++;
179 if (*index == (trb_in_seg - 1)) {
180 *index = 0;
181 *cs ^= 1;
182 }
183 }
184
cdns2_ep_inc_enq(struct cdns2_ring * ring)185 static void cdns2_ep_inc_enq(struct cdns2_ring *ring)
186 {
187 ring->free_trbs--;
188 cdns2_ep_inc_trb(&ring->enqueue, &ring->pcs, TRBS_PER_SEGMENT);
189 }
190
cdns2_ep_inc_deq(struct cdns2_ring * ring)191 static void cdns2_ep_inc_deq(struct cdns2_ring *ring)
192 {
193 ring->free_trbs++;
194 cdns2_ep_inc_trb(&ring->dequeue, &ring->ccs, TRBS_PER_SEGMENT);
195 }
196
197 /*
198 * Enable/disable LPM.
199 *
200 * If bit USBCS_LPMNYET is not set and device receive Extended Token packet,
201 * then controller answer with ACK handshake.
202 * If bit USBCS_LPMNYET is set and device receive Extended Token packet,
203 * then controller answer with NYET handshake.
204 */
cdns2_enable_l1(struct cdns2_device * pdev,int enable)205 static void cdns2_enable_l1(struct cdns2_device *pdev, int enable)
206 {
207 if (enable) {
208 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
209 writeb(LPMCLOCK_SLEEP_ENTRY, &pdev->usb_regs->lpmclock);
210 } else {
211 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
212 }
213 }
214
cdns2_get_speed(struct cdns2_device * pdev)215 static enum usb_device_speed cdns2_get_speed(struct cdns2_device *pdev)
216 {
217 u8 speed = readb(&pdev->usb_regs->speedctrl);
218
219 if (speed & SPEEDCTRL_HS)
220 return USB_SPEED_HIGH;
221 else if (speed & SPEEDCTRL_FS)
222 return USB_SPEED_FULL;
223
224 return USB_SPEED_UNKNOWN;
225 }
226
cdns2_next_trb(struct cdns2_endpoint * pep,struct cdns2_trb * trb)227 static struct cdns2_trb *cdns2_next_trb(struct cdns2_endpoint *pep,
228 struct cdns2_trb *trb)
229 {
230 if (trb == (pep->ring.trbs + (TRBS_PER_SEGMENT - 1)))
231 return pep->ring.trbs;
232 else
233 return ++trb;
234 }
235
cdns2_gadget_giveback(struct cdns2_endpoint * pep,struct cdns2_request * preq,int status)236 void cdns2_gadget_giveback(struct cdns2_endpoint *pep,
237 struct cdns2_request *preq,
238 int status)
239 {
240 struct usb_request *request = &preq->request;
241 struct cdns2_device *pdev = pep->pdev;
242
243 list_del_init(&preq->list);
244
245 if (request->status == -EINPROGRESS)
246 request->status = status;
247
248 usb_gadget_unmap_request_by_dev(pdev->dev, request, pep->dir);
249
250 /* All TRBs have finished, clear the counter. */
251 preq->finished_trb = 0;
252
253 trace_cdns2_request_giveback(preq);
254
255 if (request->complete) {
256 spin_unlock(&pdev->lock);
257 usb_gadget_giveback_request(&pep->endpoint, request);
258 spin_lock(&pdev->lock);
259 }
260
261 if (request->buf == pdev->zlp_buf)
262 cdns2_gadget_ep_free_request(&pep->endpoint, request);
263 }
264
cdns2_wa1_restore_cycle_bit(struct cdns2_endpoint * pep)265 static void cdns2_wa1_restore_cycle_bit(struct cdns2_endpoint *pep)
266 {
267 /* Work around for stale data address in TRB. */
268 if (pep->wa1_set) {
269 trace_cdns2_wa1(pep, "restore cycle bit");
270
271 pep->wa1_set = 0;
272 pep->wa1_trb_index = 0xFFFF;
273 if (pep->wa1_cycle_bit)
274 pep->wa1_trb->control |= cpu_to_le32(0x1);
275 else
276 pep->wa1_trb->control &= cpu_to_le32(~0x1);
277 }
278 }
279
cdns2_wa1_update_guard(struct cdns2_endpoint * pep,struct cdns2_trb * trb)280 static int cdns2_wa1_update_guard(struct cdns2_endpoint *pep,
281 struct cdns2_trb *trb)
282 {
283 struct cdns2_device *pdev = pep->pdev;
284
285 if (!pep->wa1_set) {
286 u32 doorbell;
287
288 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
289
290 if (doorbell) {
291 pep->wa1_cycle_bit = pep->ring.pcs ? TRB_CYCLE : 0;
292 pep->wa1_set = 1;
293 pep->wa1_trb = trb;
294 pep->wa1_trb_index = pep->ring.enqueue;
295 trace_cdns2_wa1(pep, "set guard");
296 return 0;
297 }
298 }
299 return 1;
300 }
301
cdns2_wa1_tray_restore_cycle_bit(struct cdns2_device * pdev,struct cdns2_endpoint * pep)302 static void cdns2_wa1_tray_restore_cycle_bit(struct cdns2_device *pdev,
303 struct cdns2_endpoint *pep)
304 {
305 int dma_index;
306 u32 doorbell;
307
308 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
309 dma_index = cdns2_get_dma_pos(pdev, pep);
310
311 if (!doorbell || dma_index != pep->wa1_trb_index)
312 cdns2_wa1_restore_cycle_bit(pep);
313 }
314
cdns2_prepare_ring(struct cdns2_device * pdev,struct cdns2_endpoint * pep,int num_trbs)315 static int cdns2_prepare_ring(struct cdns2_device *pdev,
316 struct cdns2_endpoint *pep,
317 int num_trbs)
318 {
319 struct cdns2_trb *link_trb = NULL;
320 int doorbell, dma_index;
321 struct cdns2_ring *ring;
322 u32 ch_bit = 0;
323
324 ring = &pep->ring;
325
326 if (num_trbs > ring->free_trbs) {
327 pep->ep_state |= EP_RING_FULL;
328 trace_cdns2_no_room_on_ring("Ring full\n");
329 return -ENOBUFS;
330 }
331
332 if ((ring->enqueue + num_trbs) >= (TRBS_PER_SEGMENT - 1)) {
333 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
334 dma_index = cdns2_get_dma_pos(pdev, pep);
335
336 /* Driver can't update LINK TRB if it is current processed. */
337 if (doorbell && dma_index == TRBS_PER_SEGMENT - 1) {
338 pep->ep_state |= EP_DEFERRED_DRDY;
339 return -ENOBUFS;
340 }
341
342 /* Update C bt in Link TRB before starting DMA. */
343 link_trb = ring->trbs + (TRBS_PER_SEGMENT - 1);
344
345 /*
346 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
347 * that DMA stuck at the LINK TRB.
348 * On the other hand, removing TRB_CHAIN for longer TRs for
349 * epXout cause that DMA stuck after handling LINK TRB.
350 * To eliminate this strange behavioral driver set TRB_CHAIN
351 * bit only for TR size > 2.
352 */
353 if (pep->type == USB_ENDPOINT_XFER_ISOC || TRBS_PER_SEGMENT > 2)
354 ch_bit = TRB_CHAIN;
355
356 link_trb->control = cpu_to_le32(((ring->pcs) ? TRB_CYCLE : 0) |
357 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
358 }
359
360 return 0;
361 }
362
cdns2_dbg_request_trbs(struct cdns2_endpoint * pep,struct cdns2_request * preq)363 static void cdns2_dbg_request_trbs(struct cdns2_endpoint *pep,
364 struct cdns2_request *preq)
365 {
366 struct cdns2_trb *link_trb = pep->ring.trbs + (TRBS_PER_SEGMENT - 1);
367 struct cdns2_trb *trb = preq->trb;
368 int num_trbs = preq->num_of_trb;
369 int i = 0;
370
371 while (i < num_trbs) {
372 trace_cdns2_queue_trb(pep, trb + i);
373 if (trb + i == link_trb) {
374 trb = pep->ring.trbs;
375 num_trbs = num_trbs - i;
376 i = 0;
377 } else {
378 i++;
379 }
380 }
381 }
382
cdns2_count_trbs(struct cdns2_endpoint * pep,u64 addr,u64 len)383 static unsigned int cdns2_count_trbs(struct cdns2_endpoint *pep,
384 u64 addr, u64 len)
385 {
386 unsigned int num_trbs = 1;
387
388 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
389 /*
390 * To speed up DMA performance address should not exceed 4KB.
391 * for high bandwidth transfer and driver will split
392 * such buffer into two TRBs.
393 */
394 num_trbs = DIV_ROUND_UP(len +
395 (addr & (TRB_MAX_ISO_BUFF_SIZE - 1)),
396 TRB_MAX_ISO_BUFF_SIZE);
397
398 if (pep->interval > 1)
399 num_trbs = pep->dir ? num_trbs * pep->interval : 1;
400 } else if (pep->dir) {
401 /*
402 * One extra link trb for IN direction.
403 * Sometimes DMA doesn't want advance to next TD and transfer
404 * hangs. This extra Link TRB force DMA to advance to next TD.
405 */
406 num_trbs++;
407 }
408
409 return num_trbs;
410 }
411
cdns2_count_sg_trbs(struct cdns2_endpoint * pep,struct usb_request * req)412 static unsigned int cdns2_count_sg_trbs(struct cdns2_endpoint *pep,
413 struct usb_request *req)
414 {
415 unsigned int i, len, full_len, num_trbs = 0;
416 struct scatterlist *sg;
417 int trb_len = 0;
418
419 full_len = req->length;
420
421 for_each_sg(req->sg, sg, req->num_sgs, i) {
422 len = sg_dma_len(sg);
423 num_trbs += cdns2_count_trbs(pep, sg_dma_address(sg), len);
424 len = min(len, full_len);
425
426 /*
427 * For HS ISO transfer TRBs should not exceed max packet size.
428 * When DMA is working, and data exceed max packet size then
429 * some data will be read in single mode instead burst mode.
430 * This behavior will drastically reduce the copying speed.
431 * To avoid this we need one or two extra TRBs.
432 * This issue occurs for UVC class with sg_supported = 1
433 * because buffers addresses are not aligned to 1024.
434 */
435 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
436 u8 temp;
437
438 trb_len += len;
439 temp = trb_len >> 10;
440
441 if (temp) {
442 if (trb_len % 1024)
443 num_trbs = num_trbs + temp;
444 else
445 num_trbs = num_trbs + temp - 1;
446
447 trb_len = trb_len - (temp << 10);
448 }
449 }
450
451 full_len -= len;
452 if (full_len == 0)
453 break;
454 }
455
456 return num_trbs;
457 }
458
459 /*
460 * Function prepares the array with optimized AXI burst value for different
461 * transfer lengths. Controller handles the final data which are less
462 * then AXI burst size as single byte transactions.
463 * e.g.:
464 * Let's assume that driver prepares trb with trb->length 700 and burst size
465 * will be set to 128. In this case the controller will handle a first 512 as
466 * single AXI transaction but the next 188 bytes will be handled
467 * as 47 separate AXI transaction.
468 * The better solution is to use the burst size equal 16 and then we will
469 * have only 25 AXI transaction (10 * 64 + 15 *4).
470 */
cdsn2_isoc_burst_opt(struct cdns2_device * pdev)471 static void cdsn2_isoc_burst_opt(struct cdns2_device *pdev)
472 {
473 int axi_burst_option[] = {1, 2, 4, 8, 16, 32, 64, 128};
474 int best_burst;
475 int array_size;
476 int opt_burst;
477 int trb_size;
478 int i, j;
479
480 array_size = ARRAY_SIZE(axi_burst_option);
481
482 for (i = 0; i <= MAX_ISO_SIZE; i++) {
483 trb_size = i / 4;
484 best_burst = trb_size ? trb_size : 1;
485
486 for (j = 0; j < array_size; j++) {
487 opt_burst = trb_size / axi_burst_option[j];
488 opt_burst += trb_size % axi_burst_option[j];
489
490 if (opt_burst < best_burst) {
491 best_burst = opt_burst;
492 pdev->burst_opt[i] = axi_burst_option[j];
493 }
494 }
495 }
496 }
497
cdns2_ep_tx_isoc(struct cdns2_endpoint * pep,struct cdns2_request * preq,int num_trbs)498 static void cdns2_ep_tx_isoc(struct cdns2_endpoint *pep,
499 struct cdns2_request *preq,
500 int num_trbs)
501 {
502 struct scatterlist *sg = NULL;
503 u32 remaining_packet_size = 0;
504 struct cdns2_trb *trb;
505 bool first_trb = true;
506 dma_addr_t trb_dma;
507 u32 trb_buff_len;
508 u32 block_length;
509 int td_idx = 0;
510 int split_size;
511 u32 full_len;
512 int enqd_len;
513 int sent_len;
514 int sg_iter;
515 u32 control;
516 int num_tds;
517 u32 length;
518
519 /*
520 * For OUT direction 1 TD per interval is enough
521 * because TRBs are not dumped by controller.
522 */
523 num_tds = pep->dir ? pep->interval : 1;
524 split_size = preq->request.num_sgs ? 1024 : 3072;
525
526 for (td_idx = 0; td_idx < num_tds; td_idx++) {
527 if (preq->request.num_sgs) {
528 sg = preq->request.sg;
529 trb_dma = sg_dma_address(sg);
530 block_length = sg_dma_len(sg);
531 } else {
532 trb_dma = preq->request.dma;
533 block_length = preq->request.length;
534 }
535
536 full_len = preq->request.length;
537 sg_iter = preq->request.num_sgs ? preq->request.num_sgs : 1;
538 remaining_packet_size = split_size;
539
540 for (enqd_len = 0; enqd_len < full_len;
541 enqd_len += trb_buff_len) {
542 if (remaining_packet_size == 0)
543 remaining_packet_size = split_size;
544
545 /*
546 * Calculate TRB length.- buffer can't across 4KB
547 * and max packet size.
548 */
549 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(trb_dma);
550 trb_buff_len = min(trb_buff_len, remaining_packet_size);
551 trb_buff_len = min(trb_buff_len, block_length);
552
553 if (trb_buff_len > full_len - enqd_len)
554 trb_buff_len = full_len - enqd_len;
555
556 control = TRB_TYPE(TRB_NORMAL);
557
558 /*
559 * For IN direction driver has to set the IOC for
560 * last TRB in last TD.
561 * For OUT direction driver must set IOC and ISP
562 * only for last TRB in each TDs.
563 */
564 if (enqd_len + trb_buff_len >= full_len || !pep->dir)
565 control |= TRB_IOC | TRB_ISP;
566
567 /*
568 * Don't give the first TRB to the hardware (by toggling
569 * the cycle bit) until we've finished creating all the
570 * other TRBs.
571 */
572 if (first_trb) {
573 first_trb = false;
574 if (pep->ring.pcs == 0)
575 control |= TRB_CYCLE;
576 } else {
577 control |= pep->ring.pcs;
578 }
579
580 if (enqd_len + trb_buff_len < full_len)
581 control |= TRB_CHAIN;
582
583 length = TRB_LEN(trb_buff_len) |
584 TRB_BURST(pep->pdev->burst_opt[trb_buff_len]);
585
586 trb = pep->ring.trbs + pep->ring.enqueue;
587 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
588 trb->length = cpu_to_le32(length);
589 trb->control = cpu_to_le32(control);
590
591 trb_dma += trb_buff_len;
592 sent_len = trb_buff_len;
593
594 if (sg && sent_len >= block_length) {
595 /* New sg entry */
596 --sg_iter;
597 sent_len -= block_length;
598 if (sg_iter != 0) {
599 sg = sg_next(sg);
600 trb_dma = sg_dma_address(sg);
601 block_length = sg_dma_len(sg);
602 }
603 }
604
605 remaining_packet_size -= trb_buff_len;
606 block_length -= sent_len;
607 preq->end_trb = pep->ring.enqueue;
608
609 cdns2_ep_inc_enq(&pep->ring);
610 }
611 }
612 }
613
cdns2_ep_tx_bulk(struct cdns2_endpoint * pep,struct cdns2_request * preq,int trbs_per_td)614 static void cdns2_ep_tx_bulk(struct cdns2_endpoint *pep,
615 struct cdns2_request *preq,
616 int trbs_per_td)
617 {
618 struct scatterlist *sg = NULL;
619 struct cdns2_ring *ring;
620 struct cdns2_trb *trb;
621 dma_addr_t trb_dma;
622 int sg_iter = 0;
623 u32 control;
624 u32 length;
625
626 if (preq->request.num_sgs) {
627 sg = preq->request.sg;
628 trb_dma = sg_dma_address(sg);
629 length = sg_dma_len(sg);
630 } else {
631 trb_dma = preq->request.dma;
632 length = preq->request.length;
633 }
634
635 ring = &pep->ring;
636
637 for (sg_iter = 0; sg_iter < trbs_per_td; sg_iter++) {
638 control = TRB_TYPE(TRB_NORMAL) | ring->pcs | TRB_ISP;
639 trb = pep->ring.trbs + ring->enqueue;
640
641 if (pep->dir && sg_iter == trbs_per_td - 1) {
642 preq->end_trb = ring->enqueue;
643 control = ring->pcs | TRB_TYPE(TRB_LINK) | TRB_CHAIN
644 | TRB_IOC;
645 cdns2_ep_inc_enq(&pep->ring);
646
647 if (ring->enqueue == 0)
648 control |= TRB_TOGGLE;
649
650 /* Point to next bad TRB. */
651 trb->buffer = cpu_to_le32(pep->ring.dma +
652 (ring->enqueue * TRB_SIZE));
653 trb->length = 0;
654 trb->control = cpu_to_le32(control);
655 break;
656 }
657
658 /*
659 * Don't give the first TRB to the hardware (by toggling
660 * the cycle bit) until we've finished creating all the
661 * other TRBs.
662 */
663 if (sg_iter == 0)
664 control = control ^ TRB_CYCLE;
665
666 /* For last TRB in TD. */
667 if (sg_iter == (trbs_per_td - (pep->dir ? 2 : 1)))
668 control |= TRB_IOC;
669 else
670 control |= TRB_CHAIN;
671
672 trb->buffer = cpu_to_le32(trb_dma);
673 trb->length = cpu_to_le32(TRB_BURST(pep->trb_burst_size) |
674 TRB_LEN(length));
675 trb->control = cpu_to_le32(control);
676
677 if (sg && sg_iter < (trbs_per_td - 1)) {
678 sg = sg_next(sg);
679 trb_dma = sg_dma_address(sg);
680 length = sg_dma_len(sg);
681 }
682
683 preq->end_trb = ring->enqueue;
684 cdns2_ep_inc_enq(&pep->ring);
685 }
686 }
687
cdns2_set_drdy(struct cdns2_device * pdev,struct cdns2_endpoint * pep)688 static void cdns2_set_drdy(struct cdns2_device *pdev,
689 struct cdns2_endpoint *pep)
690 {
691 trace_cdns2_ring(pep);
692
693 /*
694 * Memory barrier - Cycle Bit must be set before doorbell.
695 */
696 dma_wmb();
697
698 /* Clearing TRBERR and DESCMIS before setting DRDY. */
699 writel(DMA_EP_STS_TRBERR | DMA_EP_STS_DESCMIS,
700 &pdev->adma_regs->ep_sts);
701 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
702
703 if (readl(&pdev->adma_regs->ep_sts) & DMA_EP_STS_TRBERR) {
704 writel(DMA_EP_STS_TRBERR, &pdev->adma_regs->ep_sts);
705 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
706 }
707
708 trace_cdns2_doorbell_epx(pep, readl(&pdev->adma_regs->ep_traddr));
709 }
710
cdns2_prepare_first_isoc_transfer(struct cdns2_device * pdev,struct cdns2_endpoint * pep)711 static int cdns2_prepare_first_isoc_transfer(struct cdns2_device *pdev,
712 struct cdns2_endpoint *pep)
713 {
714 struct cdns2_trb *trb;
715 u32 buffer;
716 u8 hw_ccs;
717
718 if ((readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY))
719 return -EBUSY;
720
721 if (!pep->dir) {
722 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
723 writel(pep->ring.dma + pep->ring.dequeue,
724 &pdev->adma_regs->ep_traddr);
725 return 0;
726 }
727
728 /*
729 * The first packet after doorbell can be corrupted so,
730 * driver prepares 0 length packet as first packet.
731 */
732 buffer = pep->ring.dma + pep->ring.dequeue * TRB_SIZE;
733 hw_ccs = !!DMA_EP_STS_CCS(readl(&pdev->adma_regs->ep_sts));
734
735 trb = &pep->ring.trbs[TRBS_PER_SEGMENT];
736 trb->length = 0;
737 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
738 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | TRB_TYPE(TRB_NORMAL));
739
740 /*
741 * LINK TRB is used to force updating cycle bit in controller and
742 * move to correct place in transfer ring.
743 */
744 trb++;
745 trb->length = 0;
746 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
747 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) |
748 TRB_TYPE(TRB_LINK) | TRB_CHAIN);
749
750 if (hw_ccs != pep->ring.ccs)
751 trb->control |= cpu_to_le32(TRB_TOGGLE);
752
753 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
754 writel(pep->ring.dma + (TRBS_PER_SEGMENT * TRB_SIZE),
755 &pdev->adma_regs->ep_traddr);
756
757 return 0;
758 }
759
760 /* Prepare and start transfer on no-default endpoint. */
cdns2_ep_run_transfer(struct cdns2_endpoint * pep,struct cdns2_request * preq)761 static int cdns2_ep_run_transfer(struct cdns2_endpoint *pep,
762 struct cdns2_request *preq)
763 {
764 struct cdns2_device *pdev = pep->pdev;
765 struct cdns2_ring *ring;
766 u32 togle_pcs = 1;
767 int num_trbs;
768 int ret;
769
770 cdns2_select_ep(pdev, pep->endpoint.address);
771
772 if (preq->request.sg)
773 num_trbs = cdns2_count_sg_trbs(pep, &preq->request);
774 else
775 num_trbs = cdns2_count_trbs(pep, preq->request.dma,
776 preq->request.length);
777
778 ret = cdns2_prepare_ring(pdev, pep, num_trbs);
779 if (ret)
780 return ret;
781
782 ring = &pep->ring;
783 preq->start_trb = ring->enqueue;
784 preq->trb = ring->trbs + ring->enqueue;
785
786 if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) {
787 cdns2_ep_tx_isoc(pep, preq, num_trbs);
788 } else {
789 togle_pcs = cdns2_wa1_update_guard(pep, ring->trbs + ring->enqueue);
790 cdns2_ep_tx_bulk(pep, preq, num_trbs);
791 }
792
793 preq->num_of_trb = num_trbs;
794
795 /*
796 * Memory barrier - cycle bit must be set as the last operation.
797 */
798 dma_wmb();
799
800 /* Give the TD to the consumer. */
801 if (togle_pcs)
802 preq->trb->control = preq->trb->control ^ cpu_to_le32(1);
803
804 cdns2_wa1_tray_restore_cycle_bit(pdev, pep);
805 cdns2_dbg_request_trbs(pep, preq);
806
807 if (!pep->wa1_set && !(pep->ep_state & EP_STALLED) && !pep->skip) {
808 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
809 ret = cdns2_prepare_first_isoc_transfer(pdev, pep);
810 if (ret)
811 return 0;
812 }
813
814 cdns2_set_drdy(pdev, pep);
815 }
816
817 return 0;
818 }
819
820 /* Prepare and start transfer for all not started requests. */
cdns2_start_all_request(struct cdns2_device * pdev,struct cdns2_endpoint * pep)821 static int cdns2_start_all_request(struct cdns2_device *pdev,
822 struct cdns2_endpoint *pep)
823 {
824 struct cdns2_request *preq;
825 int ret;
826
827 while (!list_empty(&pep->deferred_list)) {
828 preq = cdns2_next_preq(&pep->deferred_list);
829
830 ret = cdns2_ep_run_transfer(pep, preq);
831 if (ret)
832 return ret;
833
834 list_move_tail(&preq->list, &pep->pending_list);
835 }
836
837 pep->ep_state &= ~EP_RING_FULL;
838
839 return 0;
840 }
841
842 /*
843 * Check whether trb has been handled by DMA.
844 *
845 * Endpoint must be selected before invoking this function.
846 *
847 * Returns false if request has not been handled by DMA, else returns true.
848 *
849 * SR - start ring
850 * ER - end ring
851 * DQ = ring->dequeue - dequeue position
852 * EQ = ring->enqueue - enqueue position
853 * ST = preq->start_trb - index of first TRB in transfer ring
854 * ET = preq->end_trb - index of last TRB in transfer ring
855 * CI = current_index - index of processed TRB by DMA.
856 *
857 * As first step, we check if the TRB between the ST and ET.
858 * Then, we check if cycle bit for index pep->dequeue
859 * is correct.
860 *
861 * some rules:
862 * 1. ring->dequeue never equals to current_index.
863 * 2 ring->enqueue never exceed ring->dequeue
864 * 3. exception: ring->enqueue == ring->dequeue
865 * and ring->free_trbs is zero.
866 * This case indicate that TR is full.
867 *
868 * At below two cases, the request have been handled.
869 * Case 1 - ring->dequeue < current_index
870 * SR ... EQ ... DQ ... CI ... ER
871 * SR ... DQ ... CI ... EQ ... ER
872 *
873 * Case 2 - ring->dequeue > current_index
874 * This situation takes place when CI go through the LINK TRB at the end of
875 * transfer ring.
876 * SR ... CI ... EQ ... DQ ... ER
877 */
cdns2_trb_handled(struct cdns2_endpoint * pep,struct cdns2_request * preq)878 static bool cdns2_trb_handled(struct cdns2_endpoint *pep,
879 struct cdns2_request *preq)
880 {
881 struct cdns2_device *pdev = pep->pdev;
882 struct cdns2_ring *ring;
883 struct cdns2_trb *trb;
884 int current_index = 0;
885 int handled = 0;
886 int doorbell;
887
888 ring = &pep->ring;
889 current_index = cdns2_get_dma_pos(pdev, pep);
890 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
891
892 /*
893 * Only ISO transfer can use 2 entries outside the standard
894 * Transfer Ring. First of them is used as zero length packet and the
895 * second as LINK TRB.
896 */
897 if (current_index >= TRBS_PER_SEGMENT)
898 goto finish;
899
900 /* Current trb doesn't belong to this request. */
901 if (preq->start_trb < preq->end_trb) {
902 if (ring->dequeue > preq->end_trb)
903 goto finish;
904
905 if (ring->dequeue < preq->start_trb)
906 goto finish;
907 }
908
909 if (preq->start_trb > preq->end_trb && ring->dequeue > preq->end_trb &&
910 ring->dequeue < preq->start_trb)
911 goto finish;
912
913 if (preq->start_trb == preq->end_trb && ring->dequeue != preq->end_trb)
914 goto finish;
915
916 trb = &ring->trbs[ring->dequeue];
917
918 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != ring->ccs)
919 goto finish;
920
921 if (doorbell == 1 && current_index == ring->dequeue)
922 goto finish;
923
924 /* The corner case for TRBS_PER_SEGMENT equal 2). */
925 if (TRBS_PER_SEGMENT == 2 && pep->type != USB_ENDPOINT_XFER_ISOC) {
926 handled = 1;
927 goto finish;
928 }
929
930 if (ring->enqueue == ring->dequeue &&
931 ring->free_trbs == 0) {
932 handled = 1;
933 } else if (ring->dequeue < current_index) {
934 if ((current_index == (TRBS_PER_SEGMENT - 1)) &&
935 !ring->dequeue)
936 goto finish;
937
938 handled = 1;
939 } else if (ring->dequeue > current_index) {
940 handled = 1;
941 }
942
943 finish:
944 trace_cdns2_request_handled(preq, current_index, handled);
945
946 return handled;
947 }
948
cdns2_skip_isoc_td(struct cdns2_device * pdev,struct cdns2_endpoint * pep,struct cdns2_request * preq)949 static void cdns2_skip_isoc_td(struct cdns2_device *pdev,
950 struct cdns2_endpoint *pep,
951 struct cdns2_request *preq)
952 {
953 struct cdns2_trb *trb;
954 int i;
955
956 trb = pep->ring.trbs + pep->ring.dequeue;
957
958 for (i = preq->finished_trb ; i < preq->num_of_trb; i++) {
959 preq->finished_trb++;
960 trace_cdns2_complete_trb(pep, trb);
961 cdns2_ep_inc_deq(&pep->ring);
962 trb = cdns2_next_trb(pep, trb);
963 }
964
965 cdns2_gadget_giveback(pep, preq, 0);
966 cdns2_prepare_first_isoc_transfer(pdev, pep);
967 pep->skip = false;
968 cdns2_set_drdy(pdev, pep);
969 }
970
cdns2_transfer_completed(struct cdns2_device * pdev,struct cdns2_endpoint * pep)971 static void cdns2_transfer_completed(struct cdns2_device *pdev,
972 struct cdns2_endpoint *pep)
973 {
974 struct cdns2_request *preq = NULL;
975 bool request_handled = false;
976 struct cdns2_trb *trb;
977
978 while (!list_empty(&pep->pending_list)) {
979 preq = cdns2_next_preq(&pep->pending_list);
980 trb = pep->ring.trbs + pep->ring.dequeue;
981
982 /*
983 * The TRB was changed as link TRB, and the request
984 * was handled at ep_dequeue.
985 */
986 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK &&
987 le32_to_cpu(trb->length)) {
988 trace_cdns2_complete_trb(pep, trb);
989 cdns2_ep_inc_deq(&pep->ring);
990 trb = pep->ring.trbs + pep->ring.dequeue;
991 }
992
993 /*
994 * Re-select endpoint. It could be changed by other CPU
995 * during handling usb_gadget_giveback_request.
996 */
997 cdns2_select_ep(pdev, pep->endpoint.address);
998
999 while (cdns2_trb_handled(pep, preq)) {
1000 preq->finished_trb++;
1001
1002 if (preq->finished_trb >= preq->num_of_trb)
1003 request_handled = true;
1004
1005 trb = pep->ring.trbs + pep->ring.dequeue;
1006 trace_cdns2_complete_trb(pep, trb);
1007
1008 if (pep->dir && pep->type == USB_ENDPOINT_XFER_ISOC)
1009 /*
1010 * For ISOC IN controller doens't update the
1011 * trb->length.
1012 */
1013 preq->request.actual = preq->request.length;
1014 else
1015 preq->request.actual +=
1016 TRB_LEN(le32_to_cpu(trb->length));
1017
1018 cdns2_ep_inc_deq(&pep->ring);
1019 }
1020
1021 if (request_handled) {
1022 cdns2_gadget_giveback(pep, preq, 0);
1023 request_handled = false;
1024 } else {
1025 goto prepare_next_td;
1026 }
1027
1028 if (pep->type != USB_ENDPOINT_XFER_ISOC &&
1029 TRBS_PER_SEGMENT == 2)
1030 break;
1031 }
1032
1033 prepare_next_td:
1034 if (pep->skip && preq)
1035 cdns2_skip_isoc_td(pdev, pep, preq);
1036
1037 if (!(pep->ep_state & EP_STALLED) &&
1038 !(pep->ep_state & EP_STALL_PENDING))
1039 cdns2_start_all_request(pdev, pep);
1040 }
1041
cdns2_wakeup(struct cdns2_device * pdev)1042 static void cdns2_wakeup(struct cdns2_device *pdev)
1043 {
1044 if (!pdev->may_wakeup)
1045 return;
1046
1047 /* Start driving resume signaling to indicate remote wakeup. */
1048 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_SIGRSUME);
1049 }
1050
cdns2_rearm_transfer(struct cdns2_endpoint * pep,u8 rearm)1051 static void cdns2_rearm_transfer(struct cdns2_endpoint *pep, u8 rearm)
1052 {
1053 struct cdns2_device *pdev = pep->pdev;
1054
1055 cdns2_wa1_restore_cycle_bit(pep);
1056
1057 if (rearm) {
1058 trace_cdns2_ring(pep);
1059
1060 /* Cycle Bit must be updated before arming DMA. */
1061 dma_wmb();
1062
1063 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
1064
1065 cdns2_wakeup(pdev);
1066
1067 trace_cdns2_doorbell_epx(pep,
1068 readl(&pdev->adma_regs->ep_traddr));
1069 }
1070 }
1071
cdns2_handle_epx_interrupt(struct cdns2_endpoint * pep)1072 static void cdns2_handle_epx_interrupt(struct cdns2_endpoint *pep)
1073 {
1074 struct cdns2_device *pdev = pep->pdev;
1075 u8 isoerror = 0;
1076 u32 ep_sts_reg;
1077 u32 val;
1078
1079 cdns2_select_ep(pdev, pep->endpoint.address);
1080
1081 trace_cdns2_epx_irq(pdev, pep);
1082
1083 ep_sts_reg = readl(&pdev->adma_regs->ep_sts);
1084 writel(ep_sts_reg, &pdev->adma_regs->ep_sts);
1085
1086 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
1087 u8 mult;
1088 u8 cs;
1089
1090 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1091 cs = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcs) :
1092 readb(&pdev->epx_regs->ep[pep->num - 1].rxcs);
1093 if (mult > 0)
1094 isoerror = EPX_CS_ERR(cs);
1095 }
1096
1097 /*
1098 * Sometimes ISO Error for mult=1 or mult=2 is not propagated on time
1099 * from USB module to DMA module. To protect against this driver
1100 * checks also the txcs/rxcs registers.
1101 */
1102 if ((ep_sts_reg & DMA_EP_STS_ISOERR) || isoerror) {
1103 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1104
1105 /* Wait for DBUSY cleared. */
1106 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1107 !(val & DMA_EP_STS_DBUSY), 1, 125);
1108
1109 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1110
1111 /* Wait for DFLUSH cleared. */
1112 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1113 !(val & DMA_EP_CMD_DFLUSH), 1, 10);
1114
1115 pep->skip = true;
1116 }
1117
1118 if (ep_sts_reg & DMA_EP_STS_TRBERR || pep->skip) {
1119 if (pep->ep_state & EP_STALL_PENDING &&
1120 !(ep_sts_reg & DMA_EP_STS_DESCMIS))
1121 cdns2_ep_stall_flush(pep);
1122
1123 /*
1124 * For isochronous transfer driver completes request on
1125 * IOC or on TRBERR. IOC appears only when device receive
1126 * OUT data packet. If host disable stream or lost some packet
1127 * then the only way to finish all queued transfer is to do it
1128 * on TRBERR event.
1129 */
1130 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->wa1_set) {
1131 if (!pep->dir)
1132 clear_reg_bit_32(&pdev->adma_regs->ep_cfg,
1133 DMA_EP_CFG_ENABLE);
1134
1135 cdns2_transfer_completed(pdev, pep);
1136 if (pep->ep_state & EP_DEFERRED_DRDY) {
1137 pep->ep_state &= ~EP_DEFERRED_DRDY;
1138 cdns2_set_drdy(pdev, pep);
1139 }
1140
1141 return;
1142 }
1143
1144 cdns2_transfer_completed(pdev, pep);
1145
1146 if (!(pep->ep_state & EP_STALLED) &&
1147 !(pep->ep_state & EP_STALL_PENDING)) {
1148 if (pep->ep_state & EP_DEFERRED_DRDY) {
1149 pep->ep_state &= ~EP_DEFERRED_DRDY;
1150 cdns2_start_all_request(pdev, pep);
1151 } else {
1152 cdns2_rearm_transfer(pep, pep->wa1_set);
1153 }
1154 }
1155
1156 return;
1157 }
1158
1159 if ((ep_sts_reg & DMA_EP_STS_IOC) || (ep_sts_reg & DMA_EP_STS_ISP))
1160 cdns2_transfer_completed(pdev, pep);
1161 }
1162
cdns2_disconnect_gadget(struct cdns2_device * pdev)1163 static void cdns2_disconnect_gadget(struct cdns2_device *pdev)
1164 {
1165 if (pdev->gadget_driver && pdev->gadget_driver->disconnect)
1166 pdev->gadget_driver->disconnect(&pdev->gadget);
1167 }
1168
cdns2_usb_irq_handler(int irq,void * data)1169 static irqreturn_t cdns2_usb_irq_handler(int irq, void *data)
1170 {
1171 struct cdns2_device *pdev = data;
1172 unsigned long reg_ep_ists;
1173 u8 reg_usb_irq_m;
1174 u8 reg_ext_irq_m;
1175 u8 reg_usb_irq;
1176 u8 reg_ext_irq;
1177
1178 if (pdev->in_lpm)
1179 return IRQ_NONE;
1180
1181 reg_usb_irq_m = readb(&pdev->interrupt_regs->usbien);
1182 reg_ext_irq_m = readb(&pdev->interrupt_regs->extien);
1183
1184 /* Mask all sources of interrupt. */
1185 writeb(0, &pdev->interrupt_regs->usbien);
1186 writeb(0, &pdev->interrupt_regs->extien);
1187 writel(0, &pdev->adma_regs->ep_ien);
1188
1189 /* Clear interrupt sources. */
1190 writel(0, &pdev->adma_regs->ep_sts);
1191 writeb(0, &pdev->interrupt_regs->usbirq);
1192 writeb(0, &pdev->interrupt_regs->extirq);
1193
1194 reg_ep_ists = readl(&pdev->adma_regs->ep_ists);
1195 reg_usb_irq = readb(&pdev->interrupt_regs->usbirq);
1196 reg_ext_irq = readb(&pdev->interrupt_regs->extirq);
1197
1198 if (reg_ep_ists || (reg_usb_irq & reg_usb_irq_m) ||
1199 (reg_ext_irq & reg_ext_irq_m))
1200 return IRQ_WAKE_THREAD;
1201
1202 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1203 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1204 writel(~0, &pdev->adma_regs->ep_ien);
1205
1206 return IRQ_NONE;
1207 }
1208
cdns2_thread_usb_irq_handler(struct cdns2_device * pdev)1209 static irqreturn_t cdns2_thread_usb_irq_handler(struct cdns2_device *pdev)
1210 {
1211 u8 usb_irq, ext_irq;
1212 int speed;
1213 int i;
1214
1215 ext_irq = readb(&pdev->interrupt_regs->extirq) & EXTIRQ_WAKEUP;
1216 writeb(ext_irq, &pdev->interrupt_regs->extirq);
1217
1218 usb_irq = readb(&pdev->interrupt_regs->usbirq) & USB_IEN_INIT;
1219 writeb(usb_irq, &pdev->interrupt_regs->usbirq);
1220
1221 if (!ext_irq && !usb_irq)
1222 return IRQ_NONE;
1223
1224 trace_cdns2_usb_irq(usb_irq, ext_irq);
1225
1226 if (ext_irq & EXTIRQ_WAKEUP) {
1227 if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1228 spin_unlock(&pdev->lock);
1229 pdev->gadget_driver->resume(&pdev->gadget);
1230 spin_lock(&pdev->lock);
1231 }
1232 }
1233
1234 if (usb_irq & USBIRQ_LPM) {
1235 u8 reg = readb(&pdev->usb_regs->lpmctrl);
1236
1237 /* LPM1 enter */
1238 if (!(reg & LPMCTRLLH_LPMNYET))
1239 writeb(0, &pdev->usb_regs->sleep_clkgate);
1240 }
1241
1242 if (usb_irq & USBIRQ_SUSPEND) {
1243 if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1244 spin_unlock(&pdev->lock);
1245 pdev->gadget_driver->suspend(&pdev->gadget);
1246 spin_lock(&pdev->lock);
1247 }
1248 }
1249
1250 if (usb_irq & USBIRQ_URESET) {
1251 if (pdev->gadget_driver) {
1252 pdev->dev_address = 0;
1253
1254 spin_unlock(&pdev->lock);
1255 usb_gadget_udc_reset(&pdev->gadget,
1256 pdev->gadget_driver);
1257 spin_lock(&pdev->lock);
1258
1259 /*
1260 * The USBIRQ_URESET is reported at the beginning of
1261 * reset signal. 100ms is enough time to finish reset
1262 * process. For high-speed reset procedure is completed
1263 * when controller detect HS mode.
1264 */
1265 for (i = 0; i < 100; i++) {
1266 mdelay(1);
1267 speed = cdns2_get_speed(pdev);
1268 if (speed == USB_SPEED_HIGH)
1269 break;
1270 }
1271
1272 pdev->gadget.speed = speed;
1273 cdns2_enable_l1(pdev, 0);
1274 cdns2_ep0_config(pdev);
1275 pdev->may_wakeup = 0;
1276 }
1277 }
1278
1279 if (usb_irq & USBIRQ_SUDAV) {
1280 pdev->ep0_stage = CDNS2_SETUP_STAGE;
1281 cdns2_handle_setup_packet(pdev);
1282 }
1283
1284 return IRQ_HANDLED;
1285 }
1286
1287 /* Deferred USB interrupt handler. */
cdns2_thread_irq_handler(int irq,void * data)1288 static irqreturn_t cdns2_thread_irq_handler(int irq, void *data)
1289 {
1290 struct cdns2_device *pdev = data;
1291 unsigned long dma_ep_ists;
1292 unsigned long flags;
1293 unsigned int bit;
1294
1295 local_bh_disable();
1296 spin_lock_irqsave(&pdev->lock, flags);
1297
1298 cdns2_thread_usb_irq_handler(pdev);
1299
1300 dma_ep_ists = readl(&pdev->adma_regs->ep_ists);
1301 if (!dma_ep_ists)
1302 goto unlock;
1303
1304 trace_cdns2_dma_ep_ists(dma_ep_ists);
1305
1306 /* Handle default endpoint OUT. */
1307 if (dma_ep_ists & DMA_EP_ISTS_EP_OUT0)
1308 cdns2_handle_ep0_interrupt(pdev, USB_DIR_OUT);
1309
1310 /* Handle default endpoint IN. */
1311 if (dma_ep_ists & DMA_EP_ISTS_EP_IN0)
1312 cdns2_handle_ep0_interrupt(pdev, USB_DIR_IN);
1313
1314 dma_ep_ists &= ~(DMA_EP_ISTS_EP_OUT0 | DMA_EP_ISTS_EP_IN0);
1315
1316 for_each_set_bit(bit, &dma_ep_ists, sizeof(u32) * BITS_PER_BYTE) {
1317 u8 ep_idx = bit > 16 ? (bit - 16) * 2 : (bit * 2) - 1;
1318
1319 /*
1320 * Endpoints in pdev->eps[] are held in order:
1321 * ep0, ep1out, ep1in, ep2out, ep2in... ep15out, ep15in.
1322 * but in dma_ep_ists in order:
1323 * ep0 ep1out ep2out ... ep15out ep0in ep1in .. ep15in
1324 */
1325 cdns2_handle_epx_interrupt(&pdev->eps[ep_idx]);
1326 }
1327
1328 unlock:
1329 writel(~0, &pdev->adma_regs->ep_ien);
1330 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1331 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1332
1333 spin_unlock_irqrestore(&pdev->lock, flags);
1334 local_bh_enable();
1335
1336 return IRQ_HANDLED;
1337 }
1338
1339 /* Calculates and assigns onchip memory for endpoints. */
cdns2_eps_onchip_buffer_init(struct cdns2_device * pdev)1340 static void cdns2_eps_onchip_buffer_init(struct cdns2_device *pdev)
1341 {
1342 struct cdns2_endpoint *pep;
1343 int min_buf_tx = 0;
1344 int min_buf_rx = 0;
1345 u16 tx_offset = 0;
1346 u16 rx_offset = 0;
1347 int free;
1348 int i;
1349
1350 for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
1351 pep = &pdev->eps[i];
1352
1353 if (!(pep->ep_state & EP_CLAIMED))
1354 continue;
1355
1356 if (pep->dir)
1357 min_buf_tx += pep->buffering;
1358 else
1359 min_buf_rx += pep->buffering;
1360 }
1361
1362 for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
1363 pep = &pdev->eps[i];
1364
1365 if (!(pep->ep_state & EP_CLAIMED))
1366 continue;
1367
1368 if (pep->dir) {
1369 free = pdev->onchip_tx_buf - min_buf_tx;
1370
1371 if (free + pep->buffering >= 4)
1372 free = 4;
1373 else
1374 free = free + pep->buffering;
1375
1376 min_buf_tx = min_buf_tx - pep->buffering + free;
1377
1378 pep->buffering = free;
1379
1380 writel(tx_offset,
1381 &pdev->epx_regs->txstaddr[pep->num - 1]);
1382 pdev->epx_regs->txstaddr[pep->num - 1] = tx_offset;
1383
1384 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1385 pep->name, tx_offset, pep->buffering);
1386
1387 tx_offset += pep->buffering * 1024;
1388 } else {
1389 free = pdev->onchip_rx_buf - min_buf_rx;
1390
1391 if (free + pep->buffering >= 4)
1392 free = 4;
1393 else
1394 free = free + pep->buffering;
1395
1396 min_buf_rx = min_buf_rx - pep->buffering + free;
1397
1398 pep->buffering = free;
1399 writel(rx_offset,
1400 &pdev->epx_regs->rxstaddr[pep->num - 1]);
1401
1402 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1403 pep->name, rx_offset, pep->buffering);
1404
1405 rx_offset += pep->buffering * 1024;
1406 }
1407 }
1408 }
1409
1410 /* Configure hardware endpoint. */
cdns2_ep_config(struct cdns2_endpoint * pep,bool enable)1411 static int cdns2_ep_config(struct cdns2_endpoint *pep, bool enable)
1412 {
1413 bool is_iso_ep = (pep->type == USB_ENDPOINT_XFER_ISOC);
1414 struct cdns2_device *pdev = pep->pdev;
1415 u32 max_packet_size;
1416 u8 dir = 0;
1417 u8 ep_cfg;
1418 u8 mult;
1419 u32 val;
1420 int ret;
1421
1422 switch (pep->type) {
1423 case USB_ENDPOINT_XFER_INT:
1424 ep_cfg = EPX_CON_TYPE_INT;
1425 break;
1426 case USB_ENDPOINT_XFER_BULK:
1427 ep_cfg = EPX_CON_TYPE_BULK;
1428 break;
1429 default:
1430 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1431 ep_cfg = mult << EPX_CON_ISOD_SHIFT;
1432 ep_cfg |= EPX_CON_TYPE_ISOC;
1433
1434 if (pep->dir) {
1435 set_reg_bit_8(&pdev->epx_regs->isoautoarm, BIT(pep->num));
1436 set_reg_bit_8(&pdev->epx_regs->isoautodump, BIT(pep->num));
1437 set_reg_bit_8(&pdev->epx_regs->isodctrl, BIT(pep->num));
1438 }
1439 }
1440
1441 switch (pdev->gadget.speed) {
1442 case USB_SPEED_FULL:
1443 max_packet_size = is_iso_ep ? 1023 : 64;
1444 break;
1445 case USB_SPEED_HIGH:
1446 max_packet_size = is_iso_ep ? 1024 : 512;
1447 break;
1448 default:
1449 /* All other speed are not supported. */
1450 return -EINVAL;
1451 }
1452
1453 ep_cfg |= (EPX_CON_VAL | (pep->buffering - 1));
1454
1455 if (pep->dir) {
1456 dir = FIFOCTRL_IO_TX;
1457 writew(max_packet_size, &pdev->epx_regs->txmaxpack[pep->num - 1]);
1458 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].txcon);
1459 } else {
1460 writew(max_packet_size, &pdev->epx_regs->rxmaxpack[pep->num - 1]);
1461 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].rxcon);
1462 }
1463
1464 writeb(pep->num | dir | FIFOCTRL_FIFOAUTO,
1465 &pdev->usb_regs->fifoctrl);
1466 writeb(pep->num | dir, &pdev->epx_regs->endprst);
1467 writeb(pep->num | ENDPRST_FIFORST | ENDPRST_TOGRST | dir,
1468 &pdev->epx_regs->endprst);
1469
1470 if (max_packet_size == 1024)
1471 pep->trb_burst_size = 128;
1472 else if (max_packet_size >= 512)
1473 pep->trb_burst_size = 64;
1474 else
1475 pep->trb_burst_size = 16;
1476
1477 cdns2_select_ep(pdev, pep->num | pep->dir);
1478 writel(DMA_EP_CMD_EPRST | DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
1479
1480 ret = readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1481 !(val & (DMA_EP_CMD_DFLUSH |
1482 DMA_EP_CMD_EPRST)),
1483 1, 1000);
1484
1485 if (ret)
1486 return ret;
1487
1488 writel(DMA_EP_STS_TRBERR | DMA_EP_STS_ISOERR, &pdev->adma_regs->ep_sts_en);
1489
1490 if (enable)
1491 writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg);
1492
1493 trace_cdns2_epx_hw_cfg(pdev, pep);
1494
1495 dev_dbg(pdev->dev, "Configure %s: with MPS: %08x, ep con: %02x\n",
1496 pep->name, max_packet_size, ep_cfg);
1497
1498 return 0;
1499 }
1500
cdns2_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)1501 struct usb_request *cdns2_gadget_ep_alloc_request(struct usb_ep *ep,
1502 gfp_t gfp_flags)
1503 {
1504 struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1505 struct cdns2_request *preq;
1506
1507 preq = kzalloc(sizeof(*preq), gfp_flags);
1508 if (!preq)
1509 return NULL;
1510
1511 preq->pep = pep;
1512
1513 trace_cdns2_alloc_request(preq);
1514
1515 return &preq->request;
1516 }
1517
cdns2_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)1518 void cdns2_gadget_ep_free_request(struct usb_ep *ep,
1519 struct usb_request *request)
1520 {
1521 struct cdns2_request *preq = to_cdns2_request(request);
1522
1523 trace_cdns2_free_request(preq);
1524 kfree(preq);
1525 }
1526
cdns2_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1527 static int cdns2_gadget_ep_enable(struct usb_ep *ep,
1528 const struct usb_endpoint_descriptor *desc)
1529 {
1530 u32 reg = DMA_EP_STS_EN_TRBERREN;
1531 struct cdns2_endpoint *pep;
1532 struct cdns2_device *pdev;
1533 unsigned long flags;
1534 int enable = 1;
1535 int ret = 0;
1536
1537 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
1538 !desc->wMaxPacketSize) {
1539 return -EINVAL;
1540 }
1541
1542 pep = ep_to_cdns2_ep(ep);
1543 pdev = pep->pdev;
1544
1545 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
1546 "%s is already enabled\n", pep->name))
1547 return 0;
1548
1549 spin_lock_irqsave(&pdev->lock, flags);
1550
1551 pep->type = usb_endpoint_type(desc);
1552 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1553
1554 if (pdev->gadget.speed == USB_SPEED_FULL)
1555 if (pep->type == USB_ENDPOINT_XFER_INT)
1556 pep->interval = desc->bInterval;
1557
1558 if (pep->interval > ISO_MAX_INTERVAL &&
1559 pep->type == USB_ENDPOINT_XFER_ISOC) {
1560 dev_err(pdev->dev, "ISO period is limited to %d (current: %d)\n",
1561 ISO_MAX_INTERVAL, pep->interval);
1562
1563 ret = -EINVAL;
1564 goto exit;
1565 }
1566
1567 /*
1568 * During ISO OUT traffic DMA reads Transfer Ring for the EP which has
1569 * never got doorbell.
1570 * This issue was detected only on simulation, but to avoid this issue
1571 * driver add protection against it. To fix it driver enable ISO OUT
1572 * endpoint before setting DRBL. This special treatment of ISO OUT
1573 * endpoints are recommended by controller specification.
1574 */
1575 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->dir)
1576 enable = 0;
1577
1578 ret = cdns2_alloc_tr_segment(pep);
1579 if (ret)
1580 goto exit;
1581
1582 ret = cdns2_ep_config(pep, enable);
1583 if (ret) {
1584 cdns2_free_tr_segment(pep);
1585 ret = -EINVAL;
1586 goto exit;
1587 }
1588
1589 trace_cdns2_gadget_ep_enable(pep);
1590
1591 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1592 pep->ep_state |= EP_ENABLED;
1593 pep->wa1_set = 0;
1594 pep->ring.enqueue = 0;
1595 pep->ring.dequeue = 0;
1596 reg = readl(&pdev->adma_regs->ep_sts);
1597 pep->ring.pcs = !!DMA_EP_STS_CCS(reg);
1598 pep->ring.ccs = !!DMA_EP_STS_CCS(reg);
1599
1600 writel(pep->ring.dma, &pdev->adma_regs->ep_traddr);
1601
1602 /* one TRB is reserved for link TRB used in DMULT mode*/
1603 pep->ring.free_trbs = TRBS_PER_SEGMENT - 1;
1604
1605 exit:
1606 spin_unlock_irqrestore(&pdev->lock, flags);
1607
1608 return ret;
1609 }
1610
cdns2_gadget_ep_disable(struct usb_ep * ep)1611 static int cdns2_gadget_ep_disable(struct usb_ep *ep)
1612 {
1613 struct cdns2_endpoint *pep;
1614 struct cdns2_request *preq;
1615 struct cdns2_device *pdev;
1616 unsigned long flags;
1617 int val;
1618
1619 if (!ep)
1620 return -EINVAL;
1621
1622 pep = ep_to_cdns2_ep(ep);
1623 pdev = pep->pdev;
1624
1625 if (dev_WARN_ONCE(pdev->dev, !(pep->ep_state & EP_ENABLED),
1626 "%s is already disabled\n", pep->name))
1627 return 0;
1628
1629 spin_lock_irqsave(&pdev->lock, flags);
1630
1631 trace_cdns2_gadget_ep_disable(pep);
1632
1633 cdns2_select_ep(pdev, ep->desc->bEndpointAddress);
1634
1635 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1636
1637 /*
1638 * Driver needs some time before resetting endpoint.
1639 * It need waits for clearing DBUSY bit or for timeout expired.
1640 * 10us is enough time for controller to stop transfer.
1641 */
1642 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1643 !(val & DMA_EP_STS_DBUSY), 1, 10);
1644 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
1645
1646 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1647 !(val & (DMA_EP_CMD_DFLUSH | DMA_EP_CMD_EPRST)),
1648 1, 1000);
1649
1650 while (!list_empty(&pep->pending_list)) {
1651 preq = cdns2_next_preq(&pep->pending_list);
1652 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1653 }
1654
1655 while (!list_empty(&pep->deferred_list)) {
1656 preq = cdns2_next_preq(&pep->deferred_list);
1657 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1658 }
1659
1660 ep->desc = NULL;
1661 pep->ep_state &= ~EP_ENABLED;
1662
1663 spin_unlock_irqrestore(&pdev->lock, flags);
1664
1665 return 0;
1666 }
1667
cdns2_ep_enqueue(struct cdns2_endpoint * pep,struct cdns2_request * preq,gfp_t gfp_flags)1668 static int cdns2_ep_enqueue(struct cdns2_endpoint *pep,
1669 struct cdns2_request *preq,
1670 gfp_t gfp_flags)
1671 {
1672 struct cdns2_device *pdev = pep->pdev;
1673 struct usb_request *request;
1674 int ret;
1675
1676 request = &preq->request;
1677 request->actual = 0;
1678 request->status = -EINPROGRESS;
1679
1680 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir);
1681 if (ret) {
1682 trace_cdns2_request_enqueue_error(preq);
1683 return ret;
1684 }
1685
1686 list_add_tail(&preq->list, &pep->deferred_list);
1687 trace_cdns2_request_enqueue(preq);
1688
1689 if (!(pep->ep_state & EP_STALLED) && !(pep->ep_state & EP_STALL_PENDING))
1690 cdns2_start_all_request(pdev, pep);
1691
1692 return 0;
1693 }
1694
cdns2_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1695 static int cdns2_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1696 gfp_t gfp_flags)
1697 {
1698 struct usb_request *zlp_request;
1699 struct cdns2_request *preq;
1700 struct cdns2_endpoint *pep;
1701 struct cdns2_device *pdev;
1702 unsigned long flags;
1703 int ret;
1704
1705 if (!request || !ep)
1706 return -EINVAL;
1707
1708 pep = ep_to_cdns2_ep(ep);
1709 pdev = pep->pdev;
1710
1711 if (!(pep->ep_state & EP_ENABLED)) {
1712 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1713 pep->name);
1714 return -EINVAL;
1715 }
1716
1717 spin_lock_irqsave(&pdev->lock, flags);
1718
1719 preq = to_cdns2_request(request);
1720 ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
1721
1722 if (ret == 0 && request->zero && request->length &&
1723 (request->length % ep->maxpacket == 0)) {
1724 struct cdns2_request *preq;
1725
1726 zlp_request = cdns2_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1727 zlp_request->buf = pdev->zlp_buf;
1728 zlp_request->length = 0;
1729
1730 preq = to_cdns2_request(zlp_request);
1731 ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
1732 }
1733
1734 spin_unlock_irqrestore(&pdev->lock, flags);
1735 return ret;
1736 }
1737
cdns2_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)1738 int cdns2_gadget_ep_dequeue(struct usb_ep *ep,
1739 struct usb_request *request)
1740 {
1741 struct cdns2_request *preq, *preq_temp, *cur_preq;
1742 struct cdns2_endpoint *pep;
1743 struct cdns2_trb *link_trb;
1744 u8 req_on_hw_ring = 0;
1745 unsigned long flags;
1746 u32 buffer;
1747 int val, i;
1748
1749 if (!ep || !request || !ep->desc)
1750 return -EINVAL;
1751
1752 pep = ep_to_cdns2_ep(ep);
1753 if (!pep->endpoint.desc) {
1754 dev_err(pep->pdev->dev, "%s: can't dequeue to disabled endpoint\n",
1755 pep->name);
1756 return -ESHUTDOWN;
1757 }
1758
1759 /* Requests has been dequeued during disabling endpoint. */
1760 if (!(pep->ep_state & EP_ENABLED))
1761 return 0;
1762
1763 spin_lock_irqsave(&pep->pdev->lock, flags);
1764
1765 cur_preq = to_cdns2_request(request);
1766 trace_cdns2_request_dequeue(cur_preq);
1767
1768 list_for_each_entry_safe(preq, preq_temp, &pep->pending_list, list) {
1769 if (cur_preq == preq) {
1770 req_on_hw_ring = 1;
1771 goto found;
1772 }
1773 }
1774
1775 list_for_each_entry_safe(preq, preq_temp, &pep->deferred_list, list) {
1776 if (cur_preq == preq)
1777 goto found;
1778 }
1779
1780 goto not_found;
1781
1782 found:
1783 link_trb = preq->trb;
1784
1785 /* Update ring only if removed request is on pending_req_list list. */
1786 if (req_on_hw_ring && link_trb) {
1787 /* Stop DMA */
1788 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1789
1790 /* Wait for DFLUSH cleared. */
1791 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1792 !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
1793
1794 buffer = cpu_to_le32(TRB_BUFFER(pep->ring.dma +
1795 ((preq->end_trb + 1) * TRB_SIZE)));
1796
1797 for (i = 0; i < preq->num_of_trb; i++) {
1798 link_trb->buffer = buffer;
1799 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control)
1800 & TRB_CYCLE) | TRB_CHAIN |
1801 TRB_TYPE(TRB_LINK));
1802
1803 trace_cdns2_queue_trb(pep, link_trb);
1804 link_trb = cdns2_next_trb(pep, link_trb);
1805 }
1806
1807 if (pep->wa1_trb == preq->trb)
1808 cdns2_wa1_restore_cycle_bit(pep);
1809 }
1810
1811 cdns2_gadget_giveback(pep, cur_preq, -ECONNRESET);
1812
1813 preq = cdns2_next_preq(&pep->pending_list);
1814 if (preq)
1815 cdns2_rearm_transfer(pep, 1);
1816
1817 not_found:
1818 spin_unlock_irqrestore(&pep->pdev->lock, flags);
1819 return 0;
1820 }
1821
cdns2_halt_endpoint(struct cdns2_device * pdev,struct cdns2_endpoint * pep,int value)1822 int cdns2_halt_endpoint(struct cdns2_device *pdev,
1823 struct cdns2_endpoint *pep,
1824 int value)
1825 {
1826 u8 __iomem *conf;
1827 int dir = 0;
1828
1829 if (!(pep->ep_state & EP_ENABLED))
1830 return -EPERM;
1831
1832 if (pep->dir) {
1833 dir = ENDPRST_IO_TX;
1834 conf = &pdev->epx_regs->ep[pep->num - 1].txcon;
1835 } else {
1836 conf = &pdev->epx_regs->ep[pep->num - 1].rxcon;
1837 }
1838
1839 if (!value) {
1840 struct cdns2_trb *trb = NULL;
1841 struct cdns2_request *preq;
1842 struct cdns2_trb trb_tmp;
1843
1844 preq = cdns2_next_preq(&pep->pending_list);
1845 if (preq) {
1846 trb = preq->trb;
1847 if (trb) {
1848 trb_tmp = *trb;
1849 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
1850 }
1851 }
1852
1853 trace_cdns2_ep_halt(pep, 0, 0);
1854
1855 /* Resets Sequence Number */
1856 writeb(dir | pep->num, &pdev->epx_regs->endprst);
1857 writeb(dir | ENDPRST_TOGRST | pep->num,
1858 &pdev->epx_regs->endprst);
1859
1860 clear_reg_bit_8(conf, EPX_CON_STALL);
1861
1862 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1863
1864 if (preq) {
1865 if (trb)
1866 *trb = trb_tmp;
1867
1868 cdns2_rearm_transfer(pep, 1);
1869 }
1870
1871 cdns2_start_all_request(pdev, pep);
1872 } else {
1873 trace_cdns2_ep_halt(pep, 1, 0);
1874 set_reg_bit_8(conf, EPX_CON_STALL);
1875 writeb(dir | pep->num, &pdev->epx_regs->endprst);
1876 writeb(dir | ENDPRST_FIFORST | pep->num,
1877 &pdev->epx_regs->endprst);
1878 pep->ep_state |= EP_STALLED;
1879 }
1880
1881 return 0;
1882 }
1883
1884 /* Sets/clears stall on selected endpoint. */
cdns2_gadget_ep_set_halt(struct usb_ep * ep,int value)1885 static int cdns2_gadget_ep_set_halt(struct usb_ep *ep, int value)
1886 {
1887 struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1888 struct cdns2_device *pdev = pep->pdev;
1889 struct cdns2_request *preq;
1890 unsigned long flags = 0;
1891 int ret;
1892
1893 spin_lock_irqsave(&pdev->lock, flags);
1894
1895 preq = cdns2_next_preq(&pep->pending_list);
1896 if (value && preq) {
1897 trace_cdns2_ep_busy_try_halt_again(pep);
1898 ret = -EAGAIN;
1899 goto done;
1900 }
1901
1902 if (!value)
1903 pep->ep_state &= ~EP_WEDGE;
1904
1905 ret = cdns2_halt_endpoint(pdev, pep, value);
1906
1907 done:
1908 spin_unlock_irqrestore(&pdev->lock, flags);
1909 return ret;
1910 }
1911
cdns2_gadget_ep_set_wedge(struct usb_ep * ep)1912 static int cdns2_gadget_ep_set_wedge(struct usb_ep *ep)
1913 {
1914 struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1915
1916 cdns2_gadget_ep_set_halt(ep, 1);
1917 pep->ep_state |= EP_WEDGE;
1918
1919 return 0;
1920 }
1921
1922 static struct
cdns2_find_available_ep(struct cdns2_device * pdev,struct usb_endpoint_descriptor * desc)1923 cdns2_endpoint *cdns2_find_available_ep(struct cdns2_device *pdev,
1924 struct usb_endpoint_descriptor *desc)
1925 {
1926 struct cdns2_endpoint *pep;
1927 struct usb_ep *ep;
1928 int ep_correct;
1929
1930 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
1931 unsigned long num;
1932 int ret;
1933 /* ep name pattern likes epXin or epXout. */
1934 char c[2] = {ep->name[2], '\0'};
1935
1936 ret = kstrtoul(c, 10, &num);
1937 if (ret)
1938 return ERR_PTR(ret);
1939 pep = ep_to_cdns2_ep(ep);
1940
1941 if (pep->num != num)
1942 continue;
1943
1944 ep_correct = (pep->endpoint.caps.dir_in &&
1945 usb_endpoint_dir_in(desc)) ||
1946 (pep->endpoint.caps.dir_out &&
1947 usb_endpoint_dir_out(desc));
1948
1949 if (ep_correct && !(pep->ep_state & EP_CLAIMED))
1950 return pep;
1951 }
1952
1953 return ERR_PTR(-ENOENT);
1954 }
1955
1956 /*
1957 * Function used to recognize which endpoints will be used to optimize
1958 * on-chip memory usage.
1959 */
1960 static struct
cdns2_gadget_match_ep(struct usb_gadget * gadget,struct usb_endpoint_descriptor * desc,struct usb_ss_ep_comp_descriptor * comp_desc)1961 usb_ep *cdns2_gadget_match_ep(struct usb_gadget *gadget,
1962 struct usb_endpoint_descriptor *desc,
1963 struct usb_ss_ep_comp_descriptor *comp_desc)
1964 {
1965 struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
1966 struct cdns2_endpoint *pep;
1967 unsigned long flags;
1968
1969 pep = cdns2_find_available_ep(pdev, desc);
1970 if (IS_ERR(pep)) {
1971 dev_err(pdev->dev, "no available ep\n");
1972 return NULL;
1973 }
1974
1975 spin_lock_irqsave(&pdev->lock, flags);
1976
1977 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
1978 pep->buffering = 4;
1979 else
1980 pep->buffering = 1;
1981
1982 pep->ep_state |= EP_CLAIMED;
1983 spin_unlock_irqrestore(&pdev->lock, flags);
1984
1985 return &pep->endpoint;
1986 }
1987
1988 static const struct usb_ep_ops cdns2_gadget_ep_ops = {
1989 .enable = cdns2_gadget_ep_enable,
1990 .disable = cdns2_gadget_ep_disable,
1991 .alloc_request = cdns2_gadget_ep_alloc_request,
1992 .free_request = cdns2_gadget_ep_free_request,
1993 .queue = cdns2_gadget_ep_queue,
1994 .dequeue = cdns2_gadget_ep_dequeue,
1995 .set_halt = cdns2_gadget_ep_set_halt,
1996 .set_wedge = cdns2_gadget_ep_set_wedge,
1997 };
1998
cdns2_gadget_get_frame(struct usb_gadget * gadget)1999 static int cdns2_gadget_get_frame(struct usb_gadget *gadget)
2000 {
2001 struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2002
2003 return readw(&pdev->usb_regs->frmnr);
2004 }
2005
cdns2_gadget_wakeup(struct usb_gadget * gadget)2006 static int cdns2_gadget_wakeup(struct usb_gadget *gadget)
2007 {
2008 struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2009 unsigned long flags;
2010
2011 spin_lock_irqsave(&pdev->lock, flags);
2012 cdns2_wakeup(pdev);
2013 spin_unlock_irqrestore(&pdev->lock, flags);
2014
2015 return 0;
2016 }
2017
cdns2_gadget_set_selfpowered(struct usb_gadget * gadget,int is_selfpowered)2018 static int cdns2_gadget_set_selfpowered(struct usb_gadget *gadget,
2019 int is_selfpowered)
2020 {
2021 struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2022 unsigned long flags;
2023
2024 spin_lock_irqsave(&pdev->lock, flags);
2025 pdev->is_selfpowered = !!is_selfpowered;
2026 spin_unlock_irqrestore(&pdev->lock, flags);
2027 return 0;
2028 }
2029
2030 /* Disable interrupts and begin the controller halting process. */
cdns2_quiesce(struct cdns2_device * pdev)2031 static void cdns2_quiesce(struct cdns2_device *pdev)
2032 {
2033 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2034
2035 /* Disable interrupt. */
2036 writeb(0, &pdev->interrupt_regs->extien),
2037 writeb(0, &pdev->interrupt_regs->usbien),
2038 writew(0, &pdev->adma_regs->ep_ien);
2039
2040 /* Clear interrupt line. */
2041 writeb(0x0, &pdev->interrupt_regs->usbirq);
2042 }
2043
cdns2_gadget_config(struct cdns2_device * pdev)2044 static void cdns2_gadget_config(struct cdns2_device *pdev)
2045 {
2046 cdns2_ep0_config(pdev);
2047
2048 /* Enable DMA interrupts for all endpoints. */
2049 writel(~0x0, &pdev->adma_regs->ep_ien);
2050 cdns2_enable_l1(pdev, 0);
2051 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
2052 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
2053 writel(DMA_CONF_DMULT, &pdev->adma_regs->conf);
2054 }
2055
cdns2_gadget_pullup(struct usb_gadget * gadget,int is_on)2056 static int cdns2_gadget_pullup(struct usb_gadget *gadget, int is_on)
2057 {
2058 struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2059 unsigned long flags;
2060
2061 trace_cdns2_pullup(is_on);
2062
2063 /*
2064 * Disable events handling while controller is being
2065 * enabled/disabled.
2066 */
2067 disable_irq(pdev->irq);
2068 spin_lock_irqsave(&pdev->lock, flags);
2069
2070 if (is_on) {
2071 cdns2_gadget_config(pdev);
2072 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2073 } else {
2074 cdns2_quiesce(pdev);
2075 }
2076
2077 spin_unlock_irqrestore(&pdev->lock, flags);
2078 enable_irq(pdev->irq);
2079
2080 return 0;
2081 }
2082
cdns2_gadget_udc_start(struct usb_gadget * gadget,struct usb_gadget_driver * driver)2083 static int cdns2_gadget_udc_start(struct usb_gadget *gadget,
2084 struct usb_gadget_driver *driver)
2085 {
2086 struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2087 enum usb_device_speed max_speed = driver->max_speed;
2088 unsigned long flags;
2089
2090 spin_lock_irqsave(&pdev->lock, flags);
2091 pdev->gadget_driver = driver;
2092
2093 /* Limit speed if necessary. */
2094 max_speed = min(driver->max_speed, gadget->max_speed);
2095
2096 switch (max_speed) {
2097 case USB_SPEED_FULL:
2098 writeb(SPEEDCTRL_HSDISABLE, &pdev->usb_regs->speedctrl);
2099 break;
2100 case USB_SPEED_HIGH:
2101 writeb(0, &pdev->usb_regs->speedctrl);
2102 break;
2103 default:
2104 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2105 max_speed);
2106 fallthrough;
2107 case USB_SPEED_UNKNOWN:
2108 /* Default to highspeed. */
2109 max_speed = USB_SPEED_HIGH;
2110 break;
2111 }
2112
2113 /* Reset all USB endpoints. */
2114 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2115 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
2116 &pdev->usb_regs->endprst);
2117 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->usb_regs->endprst);
2118
2119 cdns2_eps_onchip_buffer_init(pdev);
2120
2121 cdns2_gadget_config(pdev);
2122 spin_unlock_irqrestore(&pdev->lock, flags);
2123
2124 return 0;
2125 }
2126
cdns2_gadget_udc_stop(struct usb_gadget * gadget)2127 static int cdns2_gadget_udc_stop(struct usb_gadget *gadget)
2128 {
2129 struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2130 struct cdns2_endpoint *pep;
2131 u32 bEndpointAddress;
2132 struct usb_ep *ep;
2133 int val;
2134
2135 pdev->gadget_driver = NULL;
2136 pdev->gadget.speed = USB_SPEED_UNKNOWN;
2137
2138 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
2139 pep = ep_to_cdns2_ep(ep);
2140 bEndpointAddress = pep->num | pep->dir;
2141 cdns2_select_ep(pdev, bEndpointAddress);
2142 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
2143 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
2144 !(val & DMA_EP_CMD_EPRST), 1, 100);
2145 }
2146
2147 cdns2_quiesce(pdev);
2148
2149 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2150 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
2151 &pdev->epx_regs->endprst);
2152 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->epx_regs->endprst);
2153
2154 return 0;
2155 }
2156
2157 static const struct usb_gadget_ops cdns2_gadget_ops = {
2158 .get_frame = cdns2_gadget_get_frame,
2159 .wakeup = cdns2_gadget_wakeup,
2160 .set_selfpowered = cdns2_gadget_set_selfpowered,
2161 .pullup = cdns2_gadget_pullup,
2162 .udc_start = cdns2_gadget_udc_start,
2163 .udc_stop = cdns2_gadget_udc_stop,
2164 .match_ep = cdns2_gadget_match_ep,
2165 };
2166
cdns2_free_all_eps(struct cdns2_device * pdev)2167 static void cdns2_free_all_eps(struct cdns2_device *pdev)
2168 {
2169 int i;
2170
2171 for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++)
2172 cdns2_free_tr_segment(&pdev->eps[i]);
2173 }
2174
2175 /* Initializes software endpoints of gadget. */
cdns2_init_eps(struct cdns2_device * pdev)2176 static int cdns2_init_eps(struct cdns2_device *pdev)
2177 {
2178 struct cdns2_endpoint *pep;
2179 int i;
2180
2181 for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
2182 bool direction = !(i & 1); /* Start from OUT endpoint. */
2183 u8 epnum = ((i + 1) >> 1);
2184
2185 /*
2186 * Endpoints are being held in pdev->eps[] in form:
2187 * ep0, ep1out, ep1in ... ep15out, ep15in.
2188 */
2189 if (!CDNS2_IF_EP_EXIST(pdev, epnum, direction))
2190 continue;
2191
2192 pep = &pdev->eps[i];
2193 pep->pdev = pdev;
2194 pep->num = epnum;
2195 /* 0 for OUT, 1 for IN. */
2196 pep->dir = direction ? USB_DIR_IN : USB_DIR_OUT;
2197 pep->idx = i;
2198
2199 /* Ep0in and ep0out are represented by pdev->eps[0]. */
2200 if (!epnum) {
2201 int ret;
2202
2203 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2204 epnum, "BiDir");
2205
2206 cdns2_init_ep0(pdev, pep);
2207
2208 ret = cdns2_alloc_tr_segment(pep);
2209 if (ret) {
2210 dev_err(pdev->dev, "Failed to init ep0\n");
2211 return ret;
2212 }
2213 } else {
2214 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2215 epnum, !!direction ? "in" : "out");
2216 pep->endpoint.name = pep->name;
2217
2218 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
2219 pep->endpoint.ops = &cdns2_gadget_ep_ops;
2220 list_add_tail(&pep->endpoint.ep_list, &pdev->gadget.ep_list);
2221
2222 pep->endpoint.caps.dir_in = direction;
2223 pep->endpoint.caps.dir_out = !direction;
2224
2225 pep->endpoint.caps.type_iso = 1;
2226 pep->endpoint.caps.type_bulk = 1;
2227 pep->endpoint.caps.type_int = 1;
2228 }
2229
2230 pep->endpoint.name = pep->name;
2231 pep->ep_state = 0;
2232
2233 dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, "
2234 "BULK: %s, ISOC %s, SupDir IN: %s, OUT: %s\n",
2235 pep->name,
2236 (pep->endpoint.caps.type_control) ? "yes" : "no",
2237 (pep->endpoint.caps.type_int) ? "yes" : "no",
2238 (pep->endpoint.caps.type_bulk) ? "yes" : "no",
2239 (pep->endpoint.caps.type_iso) ? "yes" : "no",
2240 (pep->endpoint.caps.dir_in) ? "yes" : "no",
2241 (pep->endpoint.caps.dir_out) ? "yes" : "no");
2242
2243 INIT_LIST_HEAD(&pep->pending_list);
2244 INIT_LIST_HEAD(&pep->deferred_list);
2245 }
2246
2247 return 0;
2248 }
2249
cdns2_gadget_start(struct cdns2_device * pdev)2250 static int cdns2_gadget_start(struct cdns2_device *pdev)
2251 {
2252 u32 max_speed;
2253 void *buf;
2254 int val;
2255 int ret;
2256
2257 pdev->usb_regs = pdev->regs;
2258 pdev->ep0_regs = pdev->regs;
2259 pdev->epx_regs = pdev->regs;
2260 pdev->interrupt_regs = pdev->regs;
2261 pdev->adma_regs = pdev->regs + CDNS2_ADMA_REGS_OFFSET;
2262
2263 /* Reset controller. */
2264 set_reg_bit_8(&pdev->usb_regs->cpuctrl, CPUCTRL_SW_RST);
2265
2266 ret = readl_poll_timeout_atomic(&pdev->usb_regs->cpuctrl, val,
2267 !(val & CPUCTRL_SW_RST), 1, 10000);
2268 if (ret) {
2269 dev_err(pdev->dev, "Error: reset controller timeout\n");
2270 return -EINVAL;
2271 }
2272
2273 usb_initialize_gadget(pdev->dev, &pdev->gadget, NULL);
2274
2275 device_property_read_u16(pdev->dev, "cdns,on-chip-tx-buff-size",
2276 &pdev->onchip_tx_buf);
2277 device_property_read_u16(pdev->dev, "cdns,on-chip-rx-buff-size",
2278 &pdev->onchip_rx_buf);
2279 device_property_read_u32(pdev->dev, "cdns,avail-endpoints",
2280 &pdev->eps_supported);
2281
2282 /*
2283 * Driver assumes that each USBHS controller has at least
2284 * one IN and one OUT non control endpoint.
2285 */
2286 if (!pdev->onchip_tx_buf && !pdev->onchip_rx_buf) {
2287 ret = -EINVAL;
2288 dev_err(pdev->dev, "Invalid on-chip memory configuration\n");
2289 goto put_gadget;
2290 }
2291
2292 if (!(pdev->eps_supported & ~0x00010001)) {
2293 ret = -EINVAL;
2294 dev_err(pdev->dev, "No hardware endpoints available\n");
2295 goto put_gadget;
2296 }
2297
2298 max_speed = usb_get_maximum_speed(pdev->dev);
2299
2300 switch (max_speed) {
2301 case USB_SPEED_FULL:
2302 case USB_SPEED_HIGH:
2303 break;
2304 default:
2305 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2306 max_speed);
2307 fallthrough;
2308 case USB_SPEED_UNKNOWN:
2309 max_speed = USB_SPEED_HIGH;
2310 break;
2311 }
2312
2313 pdev->gadget.max_speed = max_speed;
2314 pdev->gadget.speed = USB_SPEED_UNKNOWN;
2315 pdev->gadget.ops = &cdns2_gadget_ops;
2316 pdev->gadget.name = "usbhs-gadget";
2317 pdev->gadget.quirk_avoids_skb_reserve = 1;
2318 pdev->gadget.irq = pdev->irq;
2319
2320 spin_lock_init(&pdev->lock);
2321 INIT_WORK(&pdev->pending_status_wq, cdns2_pending_setup_status_handler);
2322
2323 /* Initialize endpoint container. */
2324 INIT_LIST_HEAD(&pdev->gadget.ep_list);
2325 pdev->eps_dma_pool = dma_pool_create("cdns2_eps_dma_pool", pdev->dev,
2326 TR_SEG_SIZE, 8, 0);
2327 if (!pdev->eps_dma_pool) {
2328 dev_err(pdev->dev, "Failed to create TRB dma pool\n");
2329 ret = -ENOMEM;
2330 goto put_gadget;
2331 }
2332
2333 ret = cdns2_init_eps(pdev);
2334 if (ret) {
2335 dev_err(pdev->dev, "Failed to create endpoints\n");
2336 goto destroy_dma_pool;
2337 }
2338
2339 pdev->gadget.sg_supported = 1;
2340
2341 pdev->zlp_buf = kzalloc(CDNS2_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2342 if (!pdev->zlp_buf) {
2343 ret = -ENOMEM;
2344 goto destroy_dma_pool;
2345 }
2346
2347 /* Allocate memory for setup packet buffer. */
2348 buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma,
2349 GFP_DMA);
2350 pdev->ep0_preq.request.buf = buf;
2351
2352 if (!pdev->ep0_preq.request.buf) {
2353 ret = -ENOMEM;
2354 goto free_zlp_buf;
2355 }
2356
2357 /* Add USB gadget device. */
2358 ret = usb_add_gadget(&pdev->gadget);
2359 if (ret < 0) {
2360 dev_err(pdev->dev, "Failed to add gadget\n");
2361 goto free_ep0_buf;
2362 }
2363
2364 return 0;
2365
2366 free_ep0_buf:
2367 dma_free_coherent(pdev->dev, 8, pdev->ep0_preq.request.buf,
2368 pdev->ep0_preq.request.dma);
2369 free_zlp_buf:
2370 kfree(pdev->zlp_buf);
2371 destroy_dma_pool:
2372 dma_pool_destroy(pdev->eps_dma_pool);
2373 put_gadget:
2374 usb_put_gadget(&pdev->gadget);
2375
2376 return ret;
2377 }
2378
cdns2_gadget_suspend(struct cdns2_device * pdev)2379 int cdns2_gadget_suspend(struct cdns2_device *pdev)
2380 {
2381 unsigned long flags;
2382
2383 cdns2_disconnect_gadget(pdev);
2384
2385 spin_lock_irqsave(&pdev->lock, flags);
2386 pdev->gadget.speed = USB_SPEED_UNKNOWN;
2387
2388 trace_cdns2_device_state("notattached");
2389 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
2390 cdns2_enable_l1(pdev, 0);
2391
2392 /* Disable interrupt for device. */
2393 writeb(0, &pdev->interrupt_regs->usbien);
2394 writel(0, &pdev->adma_regs->ep_ien);
2395 spin_unlock_irqrestore(&pdev->lock, flags);
2396
2397 return 0;
2398 }
2399
cdns2_gadget_resume(struct cdns2_device * pdev,bool hibernated)2400 int cdns2_gadget_resume(struct cdns2_device *pdev, bool hibernated)
2401 {
2402 unsigned long flags;
2403
2404 spin_lock_irqsave(&pdev->lock, flags);
2405
2406 if (!pdev->gadget_driver) {
2407 spin_unlock_irqrestore(&pdev->lock, flags);
2408 return 0;
2409 }
2410
2411 cdns2_gadget_config(pdev);
2412
2413 if (hibernated)
2414 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2415
2416 spin_unlock_irqrestore(&pdev->lock, flags);
2417
2418 return 0;
2419 }
2420
cdns2_gadget_remove(struct cdns2_device * pdev)2421 void cdns2_gadget_remove(struct cdns2_device *pdev)
2422 {
2423 pm_runtime_mark_last_busy(pdev->dev);
2424 pm_runtime_put_autosuspend(pdev->dev);
2425
2426 usb_del_gadget(&pdev->gadget);
2427 cdns2_free_all_eps(pdev);
2428
2429 dma_pool_destroy(pdev->eps_dma_pool);
2430 kfree(pdev->zlp_buf);
2431 usb_put_gadget(&pdev->gadget);
2432 }
2433
cdns2_gadget_init(struct cdns2_device * pdev)2434 int cdns2_gadget_init(struct cdns2_device *pdev)
2435 {
2436 int ret;
2437
2438 /* Ensure 32-bit DMA Mask. */
2439 ret = dma_set_mask_and_coherent(pdev->dev, DMA_BIT_MASK(32));
2440 if (ret) {
2441 dev_err(pdev->dev, "Failed to set dma mask: %d\n", ret);
2442 return ret;
2443 }
2444
2445 pm_runtime_get_sync(pdev->dev);
2446
2447 cdsn2_isoc_burst_opt(pdev);
2448
2449 ret = cdns2_gadget_start(pdev);
2450 if (ret) {
2451 pm_runtime_put_sync(pdev->dev);
2452 return ret;
2453 }
2454
2455 /*
2456 * Because interrupt line can be shared with other components in
2457 * driver it can't use IRQF_ONESHOT flag here.
2458 */
2459 ret = devm_request_threaded_irq(pdev->dev, pdev->irq,
2460 cdns2_usb_irq_handler,
2461 cdns2_thread_irq_handler,
2462 IRQF_SHARED,
2463 dev_name(pdev->dev),
2464 pdev);
2465 if (ret)
2466 goto err0;
2467
2468 return 0;
2469
2470 err0:
2471 cdns2_gadget_remove(pdev);
2472
2473 return ret;
2474 }
2475