Lines Matching +full:num +full:- +full:transfer +full:- +full:bits
1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence USBHS-DEV Driver - gadget side.
28 #include <linux/dma-mapping.h>
36 #include "cdns2-gadget.h"
37 #include "cdns2-trace.h"
40 * set_reg_bit_32 - set bit in given 32 bits register.
42 * @mask: bits to set.
51 * clear_reg_bit_32 - clear bit in given 32 bits register.
53 * @mask: bits to clear.
61 /* Clear bit in given 8 bits register. */
68 /* Set bit in given 16 bits register. */
80 dma_index = readl(&pdev->adma_regs->ep_traddr) - pep->ring.dma; in cdns2_get_dma_pos()
93 if (pdev->selected_ep == ep) in cdns2_select_ep()
96 pdev->selected_ep = ep; in cdns2_select_ep()
97 writel(ep, &pdev->adma_regs->ep_sel); in cdns2_select_ep()
103 u32 offset = (char *)trb - (char *)pep->ring.trbs; in cdns2_trb_virt_to_dma()
105 return pep->ring.dma + offset; in cdns2_trb_virt_to_dma()
110 struct cdns2_device *pdev = pep->pdev; in cdns2_free_tr_segment()
111 struct cdns2_ring *ring = &pep->ring; in cdns2_free_tr_segment()
113 if (pep->ring.trbs) { in cdns2_free_tr_segment()
114 dma_pool_free(pdev->eps_dma_pool, ring->trbs, ring->dma); in cdns2_free_tr_segment()
119 /* Allocates Transfer Ring segment. */
122 struct cdns2_device *pdev = pep->pdev; in cdns2_alloc_tr_segment()
126 ring = &pep->ring; in cdns2_alloc_tr_segment()
128 if (!ring->trbs) { in cdns2_alloc_tr_segment()
129 ring->trbs = dma_pool_alloc(pdev->eps_dma_pool, in cdns2_alloc_tr_segment()
131 &ring->dma); in cdns2_alloc_tr_segment()
132 if (!ring->trbs) in cdns2_alloc_tr_segment()
133 return -ENOMEM; in cdns2_alloc_tr_segment()
136 memset(ring->trbs, 0, TR_SEG_SIZE); in cdns2_alloc_tr_segment()
138 if (!pep->num) in cdns2_alloc_tr_segment()
142 link_trb = (ring->trbs + (TRBS_PER_SEGMENT - 1)); in cdns2_alloc_tr_segment()
143 link_trb->buffer = cpu_to_le32(TRB_BUFFER(ring->dma)); in cdns2_alloc_tr_segment()
144 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | in cdns2_alloc_tr_segment()
156 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_stall_flush()
161 writel(DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd); in cdns2_ep_stall_flush()
164 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_ep_stall_flush()
166 pep->ep_state |= EP_STALLED; in cdns2_ep_stall_flush()
167 pep->ep_state &= ~EP_STALL_PENDING; in cdns2_ep_stall_flush()
180 if (*index == (trb_in_seg - 1)) { in cdns2_ep_inc_trb()
188 ring->free_trbs--; in cdns2_ep_inc_enq()
189 cdns2_ep_inc_trb(&ring->enqueue, &ring->pcs, TRBS_PER_SEGMENT); in cdns2_ep_inc_enq()
194 ring->free_trbs++; in cdns2_ep_inc_deq()
195 cdns2_ep_inc_trb(&ring->dequeue, &ring->ccs, TRBS_PER_SEGMENT); in cdns2_ep_inc_deq()
209 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET); in cdns2_enable_l1()
210 writeb(LPMCLOCK_SLEEP_ENTRY, &pdev->usb_regs->lpmclock); in cdns2_enable_l1()
212 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET); in cdns2_enable_l1()
218 u8 speed = readb(&pdev->usb_regs->speedctrl); in cdns2_get_speed()
231 if (trb == (pep->ring.trbs + (TRBS_PER_SEGMENT - 1))) in cdns2_next_trb()
232 return pep->ring.trbs; in cdns2_next_trb()
241 struct usb_request *request = &preq->request; in cdns2_gadget_giveback()
242 struct cdns2_device *pdev = pep->pdev; in cdns2_gadget_giveback()
244 list_del_init(&preq->list); in cdns2_gadget_giveback()
246 if (request->status == -EINPROGRESS) in cdns2_gadget_giveback()
247 request->status = status; in cdns2_gadget_giveback()
249 usb_gadget_unmap_request_by_dev(pdev->dev, request, pep->dir); in cdns2_gadget_giveback()
252 preq->finished_trb = 0; in cdns2_gadget_giveback()
256 if (request->complete) { in cdns2_gadget_giveback()
257 spin_unlock(&pdev->lock); in cdns2_gadget_giveback()
258 usb_gadget_giveback_request(&pep->endpoint, request); in cdns2_gadget_giveback()
259 spin_lock(&pdev->lock); in cdns2_gadget_giveback()
262 if (request->buf == pdev->zlp_buf) in cdns2_gadget_giveback()
263 cdns2_gadget_ep_free_request(&pep->endpoint, request); in cdns2_gadget_giveback()
269 if (pep->wa1_set) { in cdns2_wa1_restore_cycle_bit()
272 pep->wa1_set = 0; in cdns2_wa1_restore_cycle_bit()
273 pep->wa1_trb_index = 0xFFFF; in cdns2_wa1_restore_cycle_bit()
274 if (pep->wa1_cycle_bit) in cdns2_wa1_restore_cycle_bit()
275 pep->wa1_trb->control |= cpu_to_le32(0x1); in cdns2_wa1_restore_cycle_bit()
277 pep->wa1_trb->control &= cpu_to_le32(~0x1); in cdns2_wa1_restore_cycle_bit()
284 struct cdns2_device *pdev = pep->pdev; in cdns2_wa1_update_guard()
286 if (!pep->wa1_set) { in cdns2_wa1_update_guard()
289 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_wa1_update_guard()
292 pep->wa1_cycle_bit = pep->ring.pcs ? TRB_CYCLE : 0; in cdns2_wa1_update_guard()
293 pep->wa1_set = 1; in cdns2_wa1_update_guard()
294 pep->wa1_trb = trb; in cdns2_wa1_update_guard()
295 pep->wa1_trb_index = pep->ring.enqueue; in cdns2_wa1_update_guard()
309 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_wa1_tray_restore_cycle_bit()
312 if (!doorbell || dma_index != pep->wa1_trb_index) in cdns2_wa1_tray_restore_cycle_bit()
325 ring = &pep->ring; in cdns2_prepare_ring()
327 if (num_trbs > ring->free_trbs) { in cdns2_prepare_ring()
328 pep->ep_state |= EP_RING_FULL; in cdns2_prepare_ring()
330 return -ENOBUFS; in cdns2_prepare_ring()
333 if ((ring->enqueue + num_trbs) >= (TRBS_PER_SEGMENT - 1)) { in cdns2_prepare_ring()
334 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_prepare_ring()
338 if (doorbell && dma_index == TRBS_PER_SEGMENT - 1) { in cdns2_prepare_ring()
339 pep->ep_state |= EP_DEFERRED_DRDY; in cdns2_prepare_ring()
340 return -ENOBUFS; in cdns2_prepare_ring()
344 link_trb = ring->trbs + (TRBS_PER_SEGMENT - 1); in cdns2_prepare_ring()
354 if (pep->type == USB_ENDPOINT_XFER_ISOC || TRBS_PER_SEGMENT > 2) in cdns2_prepare_ring()
357 link_trb->control = cpu_to_le32(((ring->pcs) ? TRB_CYCLE : 0) | in cdns2_prepare_ring()
367 struct cdns2_trb *link_trb = pep->ring.trbs + (TRBS_PER_SEGMENT - 1); in cdns2_dbg_request_trbs()
368 struct cdns2_trb *trb = preq->trb; in cdns2_dbg_request_trbs()
369 int num_trbs = preq->num_of_trb; in cdns2_dbg_request_trbs()
375 trb = pep->ring.trbs; in cdns2_dbg_request_trbs()
376 num_trbs = num_trbs - i; in cdns2_dbg_request_trbs()
389 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_count_trbs()
392 * for high bandwidth transfer and driver will split in cdns2_count_trbs()
396 (addr & (TRB_MAX_ISO_BUFF_SIZE - 1)), in cdns2_count_trbs()
399 if (pep->interval > 1) in cdns2_count_trbs()
400 num_trbs = pep->dir ? num_trbs * pep->interval : 1; in cdns2_count_trbs()
401 } else if (pep->dir) { in cdns2_count_trbs()
404 * Sometimes DMA doesn't want advance to next TD and transfer in cdns2_count_trbs()
420 full_len = req->length; in cdns2_count_sg_trbs()
422 for_each_sg(req->sg, sg, req->num_sgs, i) { in cdns2_count_sg_trbs()
428 * For HS ISO transfer TRBs should not exceed max packet size. in cdns2_count_sg_trbs()
436 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_count_sg_trbs()
446 num_trbs = num_trbs + temp - 1; in cdns2_count_sg_trbs()
448 trb_len = trb_len - (temp << 10); in cdns2_count_sg_trbs()
452 full_len -= len; in cdns2_count_sg_trbs()
462 * transfer lengths. Controller handles the final data which are less
465 * Let's assume that driver prepares trb with trb->length 700 and burst size
493 pdev->burst_opt[i] = axi_burst_option[j]; in cdsn2_isoc_burst_opt()
524 num_tds = pep->dir ? pep->interval : 1; in cdns2_ep_tx_isoc()
525 split_size = preq->request.num_sgs ? 1024 : 3072; in cdns2_ep_tx_isoc()
528 if (preq->request.num_sgs) { in cdns2_ep_tx_isoc()
529 sg = preq->request.sg; in cdns2_ep_tx_isoc()
533 trb_dma = preq->request.dma; in cdns2_ep_tx_isoc()
534 block_length = preq->request.length; in cdns2_ep_tx_isoc()
537 full_len = preq->request.length; in cdns2_ep_tx_isoc()
538 sg_iter = preq->request.num_sgs ? preq->request.num_sgs : 1; in cdns2_ep_tx_isoc()
547 * Calculate TRB length.- buffer can't across 4KB in cdns2_ep_tx_isoc()
554 if (trb_buff_len > full_len - enqd_len) in cdns2_ep_tx_isoc()
555 trb_buff_len = full_len - enqd_len; in cdns2_ep_tx_isoc()
565 if (enqd_len + trb_buff_len >= full_len || !pep->dir) in cdns2_ep_tx_isoc()
575 if (pep->ring.pcs == 0) in cdns2_ep_tx_isoc()
578 control |= pep->ring.pcs; in cdns2_ep_tx_isoc()
585 TRB_BURST(pep->pdev->burst_opt[trb_buff_len]); in cdns2_ep_tx_isoc()
587 trb = pep->ring.trbs + pep->ring.enqueue; in cdns2_ep_tx_isoc()
588 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); in cdns2_ep_tx_isoc()
589 trb->length = cpu_to_le32(length); in cdns2_ep_tx_isoc()
590 trb->control = cpu_to_le32(control); in cdns2_ep_tx_isoc()
597 --sg_iter; in cdns2_ep_tx_isoc()
598 sent_len -= block_length; in cdns2_ep_tx_isoc()
606 remaining_packet_size -= trb_buff_len; in cdns2_ep_tx_isoc()
607 block_length -= sent_len; in cdns2_ep_tx_isoc()
608 preq->end_trb = pep->ring.enqueue; in cdns2_ep_tx_isoc()
610 cdns2_ep_inc_enq(&pep->ring); in cdns2_ep_tx_isoc()
627 if (preq->request.num_sgs) { in cdns2_ep_tx_bulk()
628 sg = preq->request.sg; in cdns2_ep_tx_bulk()
632 trb_dma = preq->request.dma; in cdns2_ep_tx_bulk()
633 length = preq->request.length; in cdns2_ep_tx_bulk()
636 ring = &pep->ring; in cdns2_ep_tx_bulk()
639 control = TRB_TYPE(TRB_NORMAL) | ring->pcs | TRB_ISP; in cdns2_ep_tx_bulk()
640 trb = pep->ring.trbs + ring->enqueue; in cdns2_ep_tx_bulk()
642 if (pep->dir && sg_iter == trbs_per_td - 1) { in cdns2_ep_tx_bulk()
643 preq->end_trb = ring->enqueue; in cdns2_ep_tx_bulk()
644 control = ring->pcs | TRB_TYPE(TRB_LINK) | TRB_CHAIN in cdns2_ep_tx_bulk()
646 cdns2_ep_inc_enq(&pep->ring); in cdns2_ep_tx_bulk()
648 if (ring->enqueue == 0) in cdns2_ep_tx_bulk()
652 trb->buffer = cpu_to_le32(pep->ring.dma + in cdns2_ep_tx_bulk()
653 (ring->enqueue * TRB_SIZE)); in cdns2_ep_tx_bulk()
654 trb->length = 0; in cdns2_ep_tx_bulk()
655 trb->control = cpu_to_le32(control); in cdns2_ep_tx_bulk()
668 if (sg_iter == (trbs_per_td - (pep->dir ? 2 : 1))) in cdns2_ep_tx_bulk()
673 trb->buffer = cpu_to_le32(trb_dma); in cdns2_ep_tx_bulk()
674 trb->length = cpu_to_le32(TRB_BURST(pep->trb_burst_size) | in cdns2_ep_tx_bulk()
676 trb->control = cpu_to_le32(control); in cdns2_ep_tx_bulk()
678 if (sg && sg_iter < (trbs_per_td - 1)) { in cdns2_ep_tx_bulk()
684 preq->end_trb = ring->enqueue; in cdns2_ep_tx_bulk()
685 cdns2_ep_inc_enq(&pep->ring); in cdns2_ep_tx_bulk()
695 * Memory barrier - Cycle Bit must be set before doorbell. in cdns2_set_drdy()
701 &pdev->adma_regs->ep_sts); in cdns2_set_drdy()
702 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd); in cdns2_set_drdy()
704 if (readl(&pdev->adma_regs->ep_sts) & DMA_EP_STS_TRBERR) { in cdns2_set_drdy()
705 writel(DMA_EP_STS_TRBERR, &pdev->adma_regs->ep_sts); in cdns2_set_drdy()
706 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd); in cdns2_set_drdy()
709 trace_cdns2_doorbell_epx(pep, readl(&pdev->adma_regs->ep_traddr)); in cdns2_set_drdy()
719 if ((readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY)) in cdns2_prepare_first_isoc_transfer()
720 return -EBUSY; in cdns2_prepare_first_isoc_transfer()
722 if (!pep->dir) { in cdns2_prepare_first_isoc_transfer()
723 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_prepare_first_isoc_transfer()
724 writel(pep->ring.dma + pep->ring.dequeue, in cdns2_prepare_first_isoc_transfer()
725 &pdev->adma_regs->ep_traddr); in cdns2_prepare_first_isoc_transfer()
733 buffer = pep->ring.dma + pep->ring.dequeue * TRB_SIZE; in cdns2_prepare_first_isoc_transfer()
734 hw_ccs = !!DMA_EP_STS_CCS(readl(&pdev->adma_regs->ep_sts)); in cdns2_prepare_first_isoc_transfer()
736 trb = &pep->ring.trbs[TRBS_PER_SEGMENT]; in cdns2_prepare_first_isoc_transfer()
737 trb->length = 0; in cdns2_prepare_first_isoc_transfer()
738 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer)); in cdns2_prepare_first_isoc_transfer()
739 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | TRB_TYPE(TRB_NORMAL)); in cdns2_prepare_first_isoc_transfer()
743 * move to correct place in transfer ring. in cdns2_prepare_first_isoc_transfer()
746 trb->length = 0; in cdns2_prepare_first_isoc_transfer()
747 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer)); in cdns2_prepare_first_isoc_transfer()
748 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | in cdns2_prepare_first_isoc_transfer()
751 if (hw_ccs != pep->ring.ccs) in cdns2_prepare_first_isoc_transfer()
752 trb->control |= cpu_to_le32(TRB_TOGGLE); in cdns2_prepare_first_isoc_transfer()
754 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_prepare_first_isoc_transfer()
755 writel(pep->ring.dma + (TRBS_PER_SEGMENT * TRB_SIZE), in cdns2_prepare_first_isoc_transfer()
756 &pdev->adma_regs->ep_traddr); in cdns2_prepare_first_isoc_transfer()
761 /* Prepare and start transfer on no-default endpoint. */
765 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_run_transfer()
771 cdns2_select_ep(pdev, pep->endpoint.address); in cdns2_ep_run_transfer()
773 if (preq->request.sg) in cdns2_ep_run_transfer()
774 num_trbs = cdns2_count_sg_trbs(pep, &preq->request); in cdns2_ep_run_transfer()
776 num_trbs = cdns2_count_trbs(pep, preq->request.dma, in cdns2_ep_run_transfer()
777 preq->request.length); in cdns2_ep_run_transfer()
783 ring = &pep->ring; in cdns2_ep_run_transfer()
784 preq->start_trb = ring->enqueue; in cdns2_ep_run_transfer()
785 preq->trb = ring->trbs + ring->enqueue; in cdns2_ep_run_transfer()
787 if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) { in cdns2_ep_run_transfer()
790 togle_pcs = cdns2_wa1_update_guard(pep, ring->trbs + ring->enqueue); in cdns2_ep_run_transfer()
794 preq->num_of_trb = num_trbs; in cdns2_ep_run_transfer()
797 * Memory barrier - cycle bit must be set as the last operation. in cdns2_ep_run_transfer()
803 preq->trb->control = preq->trb->control ^ cpu_to_le32(1); in cdns2_ep_run_transfer()
808 if (!pep->wa1_set && !(pep->ep_state & EP_STALLED) && !pep->skip) { in cdns2_ep_run_transfer()
809 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_ep_run_transfer()
821 /* Prepare and start transfer for all not started requests. */
828 while (!list_empty(&pep->deferred_list)) { in cdns2_start_all_request()
829 preq = cdns2_next_preq(&pep->deferred_list); in cdns2_start_all_request()
835 list_move_tail(&preq->list, &pep->pending_list); in cdns2_start_all_request()
838 pep->ep_state &= ~EP_RING_FULL; in cdns2_start_all_request()
850 * SR - start ring
851 * ER - end ring
852 * DQ = ring->dequeue - dequeue position
853 * EQ = ring->enqueue - enqueue position
854 * ST = preq->start_trb - index of first TRB in transfer ring
855 * ET = preq->end_trb - index of last TRB in transfer ring
856 * CI = current_index - index of processed TRB by DMA.
859 * Then, we check if cycle bit for index pep->dequeue
863 * 1. ring->dequeue never equals to current_index.
864 * 2 ring->enqueue never exceed ring->dequeue
865 * 3. exception: ring->enqueue == ring->dequeue
866 * and ring->free_trbs is zero.
870 * Case 1 - ring->dequeue < current_index
874 * Case 2 - ring->dequeue > current_index
876 * transfer ring.
882 struct cdns2_device *pdev = pep->pdev; in cdns2_trb_handled()
889 ring = &pep->ring; in cdns2_trb_handled()
891 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_trb_handled()
894 * Only ISO transfer can use 2 entries outside the standard in cdns2_trb_handled()
895 * Transfer Ring. First of them is used as zero length packet and the in cdns2_trb_handled()
902 if (preq->start_trb < preq->end_trb) { in cdns2_trb_handled()
903 if (ring->dequeue > preq->end_trb) in cdns2_trb_handled()
906 if (ring->dequeue < preq->start_trb) in cdns2_trb_handled()
910 if (preq->start_trb > preq->end_trb && ring->dequeue > preq->end_trb && in cdns2_trb_handled()
911 ring->dequeue < preq->start_trb) in cdns2_trb_handled()
914 if (preq->start_trb == preq->end_trb && ring->dequeue != preq->end_trb) in cdns2_trb_handled()
917 trb = &ring->trbs[ring->dequeue]; in cdns2_trb_handled()
919 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != ring->ccs) in cdns2_trb_handled()
922 if (doorbell == 1 && current_index == ring->dequeue) in cdns2_trb_handled()
926 if (TRBS_PER_SEGMENT == 2 && pep->type != USB_ENDPOINT_XFER_ISOC) { in cdns2_trb_handled()
931 if (ring->enqueue == ring->dequeue && in cdns2_trb_handled()
932 ring->free_trbs == 0) { in cdns2_trb_handled()
934 } else if (ring->dequeue < current_index) { in cdns2_trb_handled()
935 if ((current_index == (TRBS_PER_SEGMENT - 1)) && in cdns2_trb_handled()
936 !ring->dequeue) in cdns2_trb_handled()
940 } else if (ring->dequeue > current_index) { in cdns2_trb_handled()
957 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_skip_isoc_td()
959 for (i = preq->finished_trb ; i < preq->num_of_trb; i++) { in cdns2_skip_isoc_td()
960 preq->finished_trb++; in cdns2_skip_isoc_td()
962 cdns2_ep_inc_deq(&pep->ring); in cdns2_skip_isoc_td()
968 pep->skip = false; in cdns2_skip_isoc_td()
979 while (!list_empty(&pep->pending_list)) { in cdns2_transfer_completed()
980 preq = cdns2_next_preq(&pep->pending_list); in cdns2_transfer_completed()
981 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_transfer_completed()
987 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK && in cdns2_transfer_completed()
988 le32_to_cpu(trb->length)) { in cdns2_transfer_completed()
990 cdns2_ep_inc_deq(&pep->ring); in cdns2_transfer_completed()
991 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_transfer_completed()
995 * Re-select endpoint. It could be changed by other CPU in cdns2_transfer_completed()
998 cdns2_select_ep(pdev, pep->endpoint.address); in cdns2_transfer_completed()
1001 preq->finished_trb++; in cdns2_transfer_completed()
1003 if (preq->finished_trb >= preq->num_of_trb) in cdns2_transfer_completed()
1006 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_transfer_completed()
1009 if (pep->dir && pep->type == USB_ENDPOINT_XFER_ISOC) in cdns2_transfer_completed()
1012 * trb->length. in cdns2_transfer_completed()
1014 preq->request.actual = preq->request.length; in cdns2_transfer_completed()
1016 preq->request.actual += in cdns2_transfer_completed()
1017 TRB_LEN(le32_to_cpu(trb->length)); in cdns2_transfer_completed()
1019 cdns2_ep_inc_deq(&pep->ring); in cdns2_transfer_completed()
1029 if (pep->type != USB_ENDPOINT_XFER_ISOC && in cdns2_transfer_completed()
1035 if (pep->skip && preq) in cdns2_transfer_completed()
1038 if (!(pep->ep_state & EP_STALLED) && in cdns2_transfer_completed()
1039 !(pep->ep_state & EP_STALL_PENDING)) in cdns2_transfer_completed()
1045 if (!pdev->may_wakeup) in cdns2_wakeup()
1049 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_SIGRSUME); in cdns2_wakeup()
1054 struct cdns2_device *pdev = pep->pdev; in cdns2_rearm_transfer()
1064 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd); in cdns2_rearm_transfer()
1069 readl(&pdev->adma_regs->ep_traddr)); in cdns2_rearm_transfer()
1075 struct cdns2_device *pdev = pep->pdev; in cdns2_handle_epx_interrupt()
1080 cdns2_select_ep(pdev, pep->endpoint.address); in cdns2_handle_epx_interrupt()
1084 ep_sts_reg = readl(&pdev->adma_regs->ep_sts); in cdns2_handle_epx_interrupt()
1085 writel(ep_sts_reg, &pdev->adma_regs->ep_sts); in cdns2_handle_epx_interrupt()
1087 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_handle_epx_interrupt()
1091 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize); in cdns2_handle_epx_interrupt()
1092 cs = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcs) : in cdns2_handle_epx_interrupt()
1093 readb(&pdev->epx_regs->ep[pep->num - 1].rxcs); in cdns2_handle_epx_interrupt()
1104 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_handle_epx_interrupt()
1107 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val, in cdns2_handle_epx_interrupt()
1110 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd); in cdns2_handle_epx_interrupt()
1113 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val, in cdns2_handle_epx_interrupt()
1116 pep->skip = true; in cdns2_handle_epx_interrupt()
1119 if (ep_sts_reg & DMA_EP_STS_TRBERR || pep->skip) { in cdns2_handle_epx_interrupt()
1120 if (pep->ep_state & EP_STALL_PENDING && in cdns2_handle_epx_interrupt()
1125 * For isochronous transfer driver completes request on in cdns2_handle_epx_interrupt()
1128 * then the only way to finish all queued transfer is to do it in cdns2_handle_epx_interrupt()
1131 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->wa1_set) { in cdns2_handle_epx_interrupt()
1132 if (!pep->dir) in cdns2_handle_epx_interrupt()
1133 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, in cdns2_handle_epx_interrupt()
1137 if (pep->ep_state & EP_DEFERRED_DRDY) { in cdns2_handle_epx_interrupt()
1138 pep->ep_state &= ~EP_DEFERRED_DRDY; in cdns2_handle_epx_interrupt()
1147 if (!(pep->ep_state & EP_STALLED) && in cdns2_handle_epx_interrupt()
1148 !(pep->ep_state & EP_STALL_PENDING)) { in cdns2_handle_epx_interrupt()
1149 if (pep->ep_state & EP_DEFERRED_DRDY) { in cdns2_handle_epx_interrupt()
1150 pep->ep_state &= ~EP_DEFERRED_DRDY; in cdns2_handle_epx_interrupt()
1153 cdns2_rearm_transfer(pep, pep->wa1_set); in cdns2_handle_epx_interrupt()
1166 if (pdev->gadget_driver && pdev->gadget_driver->disconnect) in cdns2_disconnect_gadget()
1167 pdev->gadget_driver->disconnect(&pdev->gadget); in cdns2_disconnect_gadget()
1179 if (pdev->in_lpm) in cdns2_usb_irq_handler()
1182 reg_usb_irq_m = readb(&pdev->interrupt_regs->usbien); in cdns2_usb_irq_handler()
1183 reg_ext_irq_m = readb(&pdev->interrupt_regs->extien); in cdns2_usb_irq_handler()
1186 writeb(0, &pdev->interrupt_regs->usbien); in cdns2_usb_irq_handler()
1187 writeb(0, &pdev->interrupt_regs->extien); in cdns2_usb_irq_handler()
1188 writel(0, &pdev->adma_regs->ep_ien); in cdns2_usb_irq_handler()
1191 writel(0, &pdev->adma_regs->ep_sts); in cdns2_usb_irq_handler()
1192 writeb(0, &pdev->interrupt_regs->usbirq); in cdns2_usb_irq_handler()
1193 writeb(0, &pdev->interrupt_regs->extirq); in cdns2_usb_irq_handler()
1195 reg_ep_ists = readl(&pdev->adma_regs->ep_ists); in cdns2_usb_irq_handler()
1196 reg_usb_irq = readb(&pdev->interrupt_regs->usbirq); in cdns2_usb_irq_handler()
1197 reg_ext_irq = readb(&pdev->interrupt_regs->extirq); in cdns2_usb_irq_handler()
1203 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien); in cdns2_usb_irq_handler()
1204 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien); in cdns2_usb_irq_handler()
1205 writel(~0, &pdev->adma_regs->ep_ien); in cdns2_usb_irq_handler()
1216 ext_irq = readb(&pdev->interrupt_regs->extirq) & EXTIRQ_WAKEUP; in cdns2_thread_usb_irq_handler()
1217 writeb(ext_irq, &pdev->interrupt_regs->extirq); in cdns2_thread_usb_irq_handler()
1219 usb_irq = readb(&pdev->interrupt_regs->usbirq) & USB_IEN_INIT; in cdns2_thread_usb_irq_handler()
1220 writeb(usb_irq, &pdev->interrupt_regs->usbirq); in cdns2_thread_usb_irq_handler()
1228 if (pdev->gadget_driver && pdev->gadget_driver->resume) { in cdns2_thread_usb_irq_handler()
1229 spin_unlock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1230 pdev->gadget_driver->resume(&pdev->gadget); in cdns2_thread_usb_irq_handler()
1231 spin_lock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1236 u8 reg = readb(&pdev->usb_regs->lpmctrl); in cdns2_thread_usb_irq_handler()
1240 writeb(0, &pdev->usb_regs->sleep_clkgate); in cdns2_thread_usb_irq_handler()
1244 if (pdev->gadget_driver && pdev->gadget_driver->suspend) { in cdns2_thread_usb_irq_handler()
1245 spin_unlock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1246 pdev->gadget_driver->suspend(&pdev->gadget); in cdns2_thread_usb_irq_handler()
1247 spin_lock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1252 if (pdev->gadget_driver) { in cdns2_thread_usb_irq_handler()
1253 pdev->dev_address = 0; in cdns2_thread_usb_irq_handler()
1255 spin_unlock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1256 usb_gadget_udc_reset(&pdev->gadget, in cdns2_thread_usb_irq_handler()
1257 pdev->gadget_driver); in cdns2_thread_usb_irq_handler()
1258 spin_lock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1263 * process. For high-speed reset procedure is completed in cdns2_thread_usb_irq_handler()
1273 pdev->gadget.speed = speed; in cdns2_thread_usb_irq_handler()
1276 pdev->may_wakeup = 0; in cdns2_thread_usb_irq_handler()
1281 pdev->ep0_stage = CDNS2_SETUP_STAGE; in cdns2_thread_usb_irq_handler()
1297 spin_lock_irqsave(&pdev->lock, flags); in cdns2_thread_irq_handler()
1301 dma_ep_ists = readl(&pdev->adma_regs->ep_ists); in cdns2_thread_irq_handler()
1318 u8 ep_idx = bit > 16 ? (bit - 16) * 2 : (bit * 2) - 1; in cdns2_thread_irq_handler()
1321 * Endpoints in pdev->eps[] are held in order: in cdns2_thread_irq_handler()
1326 cdns2_handle_epx_interrupt(&pdev->eps[ep_idx]); in cdns2_thread_irq_handler()
1330 writel(~0, &pdev->adma_regs->ep_ien); in cdns2_thread_irq_handler()
1331 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien); in cdns2_thread_irq_handler()
1332 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien); in cdns2_thread_irq_handler()
1334 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_thread_irq_handler()
1352 pep = &pdev->eps[i]; in cdns2_eps_onchip_buffer_init()
1354 if (!(pep->ep_state & EP_CLAIMED)) in cdns2_eps_onchip_buffer_init()
1357 if (pep->dir) in cdns2_eps_onchip_buffer_init()
1358 min_buf_tx += pep->buffering; in cdns2_eps_onchip_buffer_init()
1360 min_buf_rx += pep->buffering; in cdns2_eps_onchip_buffer_init()
1364 pep = &pdev->eps[i]; in cdns2_eps_onchip_buffer_init()
1366 if (!(pep->ep_state & EP_CLAIMED)) in cdns2_eps_onchip_buffer_init()
1369 if (pep->dir) { in cdns2_eps_onchip_buffer_init()
1370 free = pdev->onchip_tx_buf - min_buf_tx; in cdns2_eps_onchip_buffer_init()
1372 if (free + pep->buffering >= 4) in cdns2_eps_onchip_buffer_init()
1375 free = free + pep->buffering; in cdns2_eps_onchip_buffer_init()
1377 min_buf_tx = min_buf_tx - pep->buffering + free; in cdns2_eps_onchip_buffer_init()
1379 pep->buffering = free; in cdns2_eps_onchip_buffer_init()
1382 &pdev->epx_regs->txstaddr[pep->num - 1]); in cdns2_eps_onchip_buffer_init()
1383 pdev->epx_regs->txstaddr[pep->num - 1] = tx_offset; in cdns2_eps_onchip_buffer_init()
1385 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n", in cdns2_eps_onchip_buffer_init()
1386 pep->name, tx_offset, pep->buffering); in cdns2_eps_onchip_buffer_init()
1388 tx_offset += pep->buffering * 1024; in cdns2_eps_onchip_buffer_init()
1390 free = pdev->onchip_rx_buf - min_buf_rx; in cdns2_eps_onchip_buffer_init()
1392 if (free + pep->buffering >= 4) in cdns2_eps_onchip_buffer_init()
1395 free = free + pep->buffering; in cdns2_eps_onchip_buffer_init()
1397 min_buf_rx = min_buf_rx - pep->buffering + free; in cdns2_eps_onchip_buffer_init()
1399 pep->buffering = free; in cdns2_eps_onchip_buffer_init()
1401 &pdev->epx_regs->rxstaddr[pep->num - 1]); in cdns2_eps_onchip_buffer_init()
1403 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n", in cdns2_eps_onchip_buffer_init()
1404 pep->name, rx_offset, pep->buffering); in cdns2_eps_onchip_buffer_init()
1406 rx_offset += pep->buffering * 1024; in cdns2_eps_onchip_buffer_init()
1414 bool is_iso_ep = (pep->type == USB_ENDPOINT_XFER_ISOC); in cdns2_ep_config()
1415 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_config()
1423 switch (pep->type) { in cdns2_ep_config()
1431 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize); in cdns2_ep_config()
1435 if (pep->dir) { in cdns2_ep_config()
1436 set_reg_bit_8(&pdev->epx_regs->isoautoarm, BIT(pep->num)); in cdns2_ep_config()
1437 set_reg_bit_8(&pdev->epx_regs->isoautodump, BIT(pep->num)); in cdns2_ep_config()
1438 set_reg_bit_8(&pdev->epx_regs->isodctrl, BIT(pep->num)); in cdns2_ep_config()
1442 switch (pdev->gadget.speed) { in cdns2_ep_config()
1451 return -EINVAL; in cdns2_ep_config()
1454 ep_cfg |= (EPX_CON_VAL | (pep->buffering - 1)); in cdns2_ep_config()
1456 if (pep->dir) { in cdns2_ep_config()
1458 writew(max_packet_size, &pdev->epx_regs->txmaxpack[pep->num - 1]); in cdns2_ep_config()
1459 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].txcon); in cdns2_ep_config()
1461 writew(max_packet_size, &pdev->epx_regs->rxmaxpack[pep->num - 1]); in cdns2_ep_config()
1462 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].rxcon); in cdns2_ep_config()
1465 writeb(pep->num | dir | FIFOCTRL_FIFOAUTO, in cdns2_ep_config()
1466 &pdev->usb_regs->fifoctrl); in cdns2_ep_config()
1467 writeb(pep->num | dir, &pdev->epx_regs->endprst); in cdns2_ep_config()
1468 writeb(pep->num | ENDPRST_FIFORST | ENDPRST_TOGRST | dir, in cdns2_ep_config()
1469 &pdev->epx_regs->endprst); in cdns2_ep_config()
1472 pep->trb_burst_size = 128; in cdns2_ep_config()
1474 pep->trb_burst_size = 64; in cdns2_ep_config()
1476 pep->trb_burst_size = 16; in cdns2_ep_config()
1478 cdns2_select_ep(pdev, pep->num | pep->dir); in cdns2_ep_config()
1479 writel(DMA_EP_CMD_EPRST | DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd); in cdns2_ep_config()
1481 ret = readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_ep_config()
1489 writel(DMA_EP_STS_TRBERR | DMA_EP_STS_ISOERR, &pdev->adma_regs->ep_sts_en); in cdns2_ep_config()
1492 writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg); in cdns2_ep_config()
1496 dev_dbg(pdev->dev, "Configure %s: with MPS: %08x, ep con: %02x\n", in cdns2_ep_config()
1497 pep->name, max_packet_size, ep_cfg); in cdns2_ep_config()
1512 preq->pep = pep; in cdns2_gadget_ep_alloc_request()
1516 return &preq->request; in cdns2_gadget_ep_alloc_request()
1538 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT || in cdns2_gadget_ep_enable()
1539 !desc->wMaxPacketSize) { in cdns2_gadget_ep_enable()
1540 return -EINVAL; in cdns2_gadget_ep_enable()
1544 pdev = pep->pdev; in cdns2_gadget_ep_enable()
1546 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED, in cdns2_gadget_ep_enable()
1547 "%s is already enabled\n", pep->name)) in cdns2_gadget_ep_enable()
1550 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_enable()
1552 pep->type = usb_endpoint_type(desc); in cdns2_gadget_ep_enable()
1553 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; in cdns2_gadget_ep_enable()
1555 if (pdev->gadget.speed == USB_SPEED_FULL) in cdns2_gadget_ep_enable()
1556 if (pep->type == USB_ENDPOINT_XFER_INT) in cdns2_gadget_ep_enable()
1557 pep->interval = desc->bInterval; in cdns2_gadget_ep_enable()
1559 if (pep->interval > ISO_MAX_INTERVAL && in cdns2_gadget_ep_enable()
1560 pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_gadget_ep_enable()
1561 dev_err(pdev->dev, "ISO period is limited to %d (current: %d)\n", in cdns2_gadget_ep_enable()
1562 ISO_MAX_INTERVAL, pep->interval); in cdns2_gadget_ep_enable()
1564 ret = -EINVAL; in cdns2_gadget_ep_enable()
1569 * During ISO OUT traffic DMA reads Transfer Ring for the EP which has in cdns2_gadget_ep_enable()
1576 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->dir) in cdns2_gadget_ep_enable()
1586 ret = -EINVAL; in cdns2_gadget_ep_enable()
1592 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING); in cdns2_gadget_ep_enable()
1593 pep->ep_state |= EP_ENABLED; in cdns2_gadget_ep_enable()
1594 pep->wa1_set = 0; in cdns2_gadget_ep_enable()
1595 pep->ring.enqueue = 0; in cdns2_gadget_ep_enable()
1596 pep->ring.dequeue = 0; in cdns2_gadget_ep_enable()
1597 reg = readl(&pdev->adma_regs->ep_sts); in cdns2_gadget_ep_enable()
1598 pep->ring.pcs = !!DMA_EP_STS_CCS(reg); in cdns2_gadget_ep_enable()
1599 pep->ring.ccs = !!DMA_EP_STS_CCS(reg); in cdns2_gadget_ep_enable()
1601 writel(pep->ring.dma, &pdev->adma_regs->ep_traddr); in cdns2_gadget_ep_enable()
1604 pep->ring.free_trbs = TRBS_PER_SEGMENT - 1; in cdns2_gadget_ep_enable()
1607 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_enable()
1621 return -EINVAL; in cdns2_gadget_ep_disable()
1624 pdev = pep->pdev; in cdns2_gadget_ep_disable()
1626 if (dev_WARN_ONCE(pdev->dev, !(pep->ep_state & EP_ENABLED), in cdns2_gadget_ep_disable()
1627 "%s is already disabled\n", pep->name)) in cdns2_gadget_ep_disable()
1630 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_disable()
1634 cdns2_select_ep(pdev, ep->desc->bEndpointAddress); in cdns2_gadget_ep_disable()
1636 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_gadget_ep_disable()
1641 * 10us is enough time for controller to stop transfer. in cdns2_gadget_ep_disable()
1643 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val, in cdns2_gadget_ep_disable()
1645 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd); in cdns2_gadget_ep_disable()
1647 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_gadget_ep_disable()
1651 while (!list_empty(&pep->pending_list)) { in cdns2_gadget_ep_disable()
1652 preq = cdns2_next_preq(&pep->pending_list); in cdns2_gadget_ep_disable()
1653 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN); in cdns2_gadget_ep_disable()
1656 while (!list_empty(&pep->deferred_list)) { in cdns2_gadget_ep_disable()
1657 preq = cdns2_next_preq(&pep->deferred_list); in cdns2_gadget_ep_disable()
1658 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN); in cdns2_gadget_ep_disable()
1661 ep->desc = NULL; in cdns2_gadget_ep_disable()
1662 pep->ep_state &= ~EP_ENABLED; in cdns2_gadget_ep_disable()
1664 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_disable()
1673 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_enqueue()
1677 request = &preq->request; in cdns2_ep_enqueue()
1678 request->actual = 0; in cdns2_ep_enqueue()
1679 request->status = -EINPROGRESS; in cdns2_ep_enqueue()
1681 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir); in cdns2_ep_enqueue()
1687 list_add_tail(&preq->list, &pep->deferred_list); in cdns2_ep_enqueue()
1690 if (!(pep->ep_state & EP_STALLED) && !(pep->ep_state & EP_STALL_PENDING)) in cdns2_ep_enqueue()
1707 return -EINVAL; in cdns2_gadget_ep_queue()
1710 pdev = pep->pdev; in cdns2_gadget_ep_queue()
1712 if (!(pep->ep_state & EP_ENABLED)) { in cdns2_gadget_ep_queue()
1713 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n", in cdns2_gadget_ep_queue()
1714 pep->name); in cdns2_gadget_ep_queue()
1715 return -EINVAL; in cdns2_gadget_ep_queue()
1718 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_queue()
1723 if (ret == 0 && request->zero && request->length && in cdns2_gadget_ep_queue()
1724 (request->length % ep->maxpacket == 0)) { in cdns2_gadget_ep_queue()
1728 zlp_request->buf = pdev->zlp_buf; in cdns2_gadget_ep_queue()
1729 zlp_request->length = 0; in cdns2_gadget_ep_queue()
1735 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_queue()
1750 if (!ep || !request || !ep->desc) in cdns2_gadget_ep_dequeue()
1751 return -EINVAL; in cdns2_gadget_ep_dequeue()
1754 if (!pep->endpoint.desc) { in cdns2_gadget_ep_dequeue()
1755 dev_err(pep->pdev->dev, "%s: can't dequeue to disabled endpoint\n", in cdns2_gadget_ep_dequeue()
1756 pep->name); in cdns2_gadget_ep_dequeue()
1757 return -ESHUTDOWN; in cdns2_gadget_ep_dequeue()
1761 if (!(pep->ep_state & EP_ENABLED)) in cdns2_gadget_ep_dequeue()
1764 spin_lock_irqsave(&pep->pdev->lock, flags); in cdns2_gadget_ep_dequeue()
1769 list_for_each_entry_safe(preq, preq_temp, &pep->pending_list, list) { in cdns2_gadget_ep_dequeue()
1776 list_for_each_entry_safe(preq, preq_temp, &pep->deferred_list, list) { in cdns2_gadget_ep_dequeue()
1784 link_trb = preq->trb; in cdns2_gadget_ep_dequeue()
1789 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd); in cdns2_gadget_ep_dequeue()
1792 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val, in cdns2_gadget_ep_dequeue()
1795 buffer = cpu_to_le32(TRB_BUFFER(pep->ring.dma + in cdns2_gadget_ep_dequeue()
1796 ((preq->end_trb + 1) * TRB_SIZE))); in cdns2_gadget_ep_dequeue()
1798 for (i = 0; i < preq->num_of_trb; i++) { in cdns2_gadget_ep_dequeue()
1799 link_trb->buffer = buffer; in cdns2_gadget_ep_dequeue()
1800 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) in cdns2_gadget_ep_dequeue()
1808 if (pep->wa1_trb == preq->trb) in cdns2_gadget_ep_dequeue()
1812 cdns2_gadget_giveback(pep, cur_preq, -ECONNRESET); in cdns2_gadget_ep_dequeue()
1814 preq = cdns2_next_preq(&pep->pending_list); in cdns2_gadget_ep_dequeue()
1819 spin_unlock_irqrestore(&pep->pdev->lock, flags); in cdns2_gadget_ep_dequeue()
1830 if (!(pep->ep_state & EP_ENABLED)) in cdns2_halt_endpoint()
1831 return -EPERM; in cdns2_halt_endpoint()
1833 if (pep->dir) { in cdns2_halt_endpoint()
1835 conf = &pdev->epx_regs->ep[pep->num - 1].txcon; in cdns2_halt_endpoint()
1837 conf = &pdev->epx_regs->ep[pep->num - 1].rxcon; in cdns2_halt_endpoint()
1845 preq = cdns2_next_preq(&pep->pending_list); in cdns2_halt_endpoint()
1847 trb = preq->trb; in cdns2_halt_endpoint()
1850 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE); in cdns2_halt_endpoint()
1857 writeb(dir | pep->num, &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1858 writeb(dir | ENDPRST_TOGRST | pep->num, in cdns2_halt_endpoint()
1859 &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1863 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING); in cdns2_halt_endpoint()
1876 writeb(dir | pep->num, &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1877 writeb(dir | ENDPRST_FIFORST | pep->num, in cdns2_halt_endpoint()
1878 &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1879 pep->ep_state |= EP_STALLED; in cdns2_halt_endpoint()
1889 struct cdns2_device *pdev = pep->pdev; in cdns2_gadget_ep_set_halt()
1894 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_set_halt()
1896 preq = cdns2_next_preq(&pep->pending_list); in cdns2_gadget_ep_set_halt()
1899 ret = -EAGAIN; in cdns2_gadget_ep_set_halt()
1904 pep->ep_state &= ~EP_WEDGE; in cdns2_gadget_ep_set_halt()
1909 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_set_halt()
1918 pep->ep_state |= EP_WEDGE; in cdns2_gadget_ep_set_wedge()
1931 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) { in cdns2_find_available_ep()
1932 unsigned long num; in cdns2_find_available_ep() local
1935 char c[2] = {ep->name[2], '\0'}; in cdns2_find_available_ep()
1937 ret = kstrtoul(c, 10, &num); in cdns2_find_available_ep()
1942 if (pep->num != num) in cdns2_find_available_ep()
1945 ep_correct = (pep->endpoint.caps.dir_in && in cdns2_find_available_ep()
1947 (pep->endpoint.caps.dir_out && in cdns2_find_available_ep()
1950 if (ep_correct && !(pep->ep_state & EP_CLAIMED)) in cdns2_find_available_ep()
1954 return ERR_PTR(-ENOENT); in cdns2_find_available_ep()
1959 * on-chip memory usage.
1972 dev_err(pdev->dev, "no available ep\n"); in cdns2_gadget_match_ep()
1976 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_match_ep()
1979 pep->buffering = 4; in cdns2_gadget_match_ep()
1981 pep->buffering = 1; in cdns2_gadget_match_ep()
1983 pep->ep_state |= EP_CLAIMED; in cdns2_gadget_match_ep()
1984 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_match_ep()
1986 return &pep->endpoint; in cdns2_gadget_match_ep()
2004 return readw(&pdev->usb_regs->frmnr); in cdns2_gadget_get_frame()
2012 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_wakeup()
2014 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_wakeup()
2025 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_set_selfpowered()
2026 pdev->is_selfpowered = !!is_selfpowered; in cdns2_gadget_set_selfpowered()
2027 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_set_selfpowered()
2034 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON); in cdns2_quiesce()
2037 writeb(0, &pdev->interrupt_regs->extien); in cdns2_quiesce()
2038 writeb(0, &pdev->interrupt_regs->usbien); in cdns2_quiesce()
2039 writew(0, &pdev->adma_regs->ep_ien); in cdns2_quiesce()
2042 writeb(0x0, &pdev->interrupt_regs->usbirq); in cdns2_quiesce()
2050 writel(~0x0, &pdev->adma_regs->ep_ien); in cdns2_gadget_config()
2052 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien); in cdns2_gadget_config()
2053 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien); in cdns2_gadget_config()
2054 writel(DMA_CONF_DMULT, &pdev->adma_regs->conf); in cdns2_gadget_config()
2068 disable_irq(pdev->irq); in cdns2_gadget_pullup()
2069 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_pullup()
2073 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON); in cdns2_gadget_pullup()
2078 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_pullup()
2079 enable_irq(pdev->irq); in cdns2_gadget_pullup()
2088 enum usb_device_speed max_speed = driver->max_speed; in cdns2_gadget_udc_start()
2091 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_udc_start()
2092 pdev->gadget_driver = driver; in cdns2_gadget_udc_start()
2095 max_speed = min(driver->max_speed, gadget->max_speed); in cdns2_gadget_udc_start()
2099 writeb(SPEEDCTRL_HSDISABLE, &pdev->usb_regs->speedctrl); in cdns2_gadget_udc_start()
2102 writeb(0, &pdev->usb_regs->speedctrl); in cdns2_gadget_udc_start()
2105 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n", in cdns2_gadget_udc_start()
2115 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst); in cdns2_gadget_udc_start()
2117 &pdev->usb_regs->endprst); in cdns2_gadget_udc_start()
2118 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->usb_regs->endprst); in cdns2_gadget_udc_start()
2123 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_udc_start()
2136 pdev->gadget_driver = NULL; in cdns2_gadget_udc_stop()
2137 pdev->gadget.speed = USB_SPEED_UNKNOWN; in cdns2_gadget_udc_stop()
2139 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) { in cdns2_gadget_udc_stop()
2141 bEndpointAddress = pep->num | pep->dir; in cdns2_gadget_udc_stop()
2143 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd); in cdns2_gadget_udc_stop()
2144 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_gadget_udc_stop()
2150 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst); in cdns2_gadget_udc_stop()
2152 &pdev->epx_regs->endprst); in cdns2_gadget_udc_stop()
2153 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->epx_regs->endprst); in cdns2_gadget_udc_stop()
2173 cdns2_free_tr_segment(&pdev->eps[i]); in cdns2_free_all_eps()
2187 * Endpoints are being held in pdev->eps[] in form: in cdns2_init_eps()
2193 pep = &pdev->eps[i]; in cdns2_init_eps()
2194 pep->pdev = pdev; in cdns2_init_eps()
2195 pep->num = epnum; in cdns2_init_eps()
2197 pep->dir = direction ? USB_DIR_IN : USB_DIR_OUT; in cdns2_init_eps()
2198 pep->idx = i; in cdns2_init_eps()
2200 /* Ep0in and ep0out are represented by pdev->eps[0]. */ in cdns2_init_eps()
2204 snprintf(pep->name, sizeof(pep->name), "ep%d%s", in cdns2_init_eps()
2211 dev_err(pdev->dev, "Failed to init ep0\n"); in cdns2_init_eps()
2215 snprintf(pep->name, sizeof(pep->name), "ep%d%s", in cdns2_init_eps()
2217 pep->endpoint.name = pep->name; in cdns2_init_eps()
2219 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024); in cdns2_init_eps()
2220 pep->endpoint.ops = &cdns2_gadget_ep_ops; in cdns2_init_eps()
2221 list_add_tail(&pep->endpoint.ep_list, &pdev->gadget.ep_list); in cdns2_init_eps()
2223 pep->endpoint.caps.dir_in = direction; in cdns2_init_eps()
2224 pep->endpoint.caps.dir_out = !direction; in cdns2_init_eps()
2226 pep->endpoint.caps.type_iso = 1; in cdns2_init_eps()
2227 pep->endpoint.caps.type_bulk = 1; in cdns2_init_eps()
2228 pep->endpoint.caps.type_int = 1; in cdns2_init_eps()
2231 pep->endpoint.name = pep->name; in cdns2_init_eps()
2232 pep->ep_state = 0; in cdns2_init_eps()
2234 dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, " in cdns2_init_eps()
2236 pep->name, in cdns2_init_eps()
2237 str_yes_no(pep->endpoint.caps.type_control), in cdns2_init_eps()
2238 str_yes_no(pep->endpoint.caps.type_int), in cdns2_init_eps()
2239 str_yes_no(pep->endpoint.caps.type_bulk), in cdns2_init_eps()
2240 str_yes_no(pep->endpoint.caps.type_iso), in cdns2_init_eps()
2241 str_yes_no(pep->endpoint.caps.dir_in), in cdns2_init_eps()
2242 str_yes_no(pep->endpoint.caps.dir_out)); in cdns2_init_eps()
2244 INIT_LIST_HEAD(&pep->pending_list); in cdns2_init_eps()
2245 INIT_LIST_HEAD(&pep->deferred_list); in cdns2_init_eps()
2257 pdev->usb_regs = pdev->regs; in cdns2_gadget_start()
2258 pdev->ep0_regs = pdev->regs; in cdns2_gadget_start()
2259 pdev->epx_regs = pdev->regs; in cdns2_gadget_start()
2260 pdev->interrupt_regs = pdev->regs; in cdns2_gadget_start()
2261 pdev->adma_regs = pdev->regs + CDNS2_ADMA_REGS_OFFSET; in cdns2_gadget_start()
2265 &pdev->usb_regs->cpuctrl); in cdns2_gadget_start()
2268 usb_initialize_gadget(pdev->dev, &pdev->gadget, NULL); in cdns2_gadget_start()
2270 device_property_read_u16(pdev->dev, "cdns,on-chip-tx-buff-size", in cdns2_gadget_start()
2271 &pdev->onchip_tx_buf); in cdns2_gadget_start()
2272 device_property_read_u16(pdev->dev, "cdns,on-chip-rx-buff-size", in cdns2_gadget_start()
2273 &pdev->onchip_rx_buf); in cdns2_gadget_start()
2274 device_property_read_u32(pdev->dev, "cdns,avail-endpoints", in cdns2_gadget_start()
2275 &pdev->eps_supported); in cdns2_gadget_start()
2281 if (!pdev->onchip_tx_buf && !pdev->onchip_rx_buf) { in cdns2_gadget_start()
2282 ret = -EINVAL; in cdns2_gadget_start()
2283 dev_err(pdev->dev, "Invalid on-chip memory configuration\n"); in cdns2_gadget_start()
2287 if (!(pdev->eps_supported & ~0x00010001)) { in cdns2_gadget_start()
2288 ret = -EINVAL; in cdns2_gadget_start()
2289 dev_err(pdev->dev, "No hardware endpoints available\n"); in cdns2_gadget_start()
2293 max_speed = usb_get_maximum_speed(pdev->dev); in cdns2_gadget_start()
2300 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n", in cdns2_gadget_start()
2308 pdev->gadget.max_speed = max_speed; in cdns2_gadget_start()
2309 pdev->gadget.speed = USB_SPEED_UNKNOWN; in cdns2_gadget_start()
2310 pdev->gadget.ops = &cdns2_gadget_ops; in cdns2_gadget_start()
2311 pdev->gadget.name = "usbhs-gadget"; in cdns2_gadget_start()
2312 pdev->gadget.quirk_avoids_skb_reserve = 1; in cdns2_gadget_start()
2313 pdev->gadget.irq = pdev->irq; in cdns2_gadget_start()
2315 spin_lock_init(&pdev->lock); in cdns2_gadget_start()
2316 INIT_WORK(&pdev->pending_status_wq, cdns2_pending_setup_status_handler); in cdns2_gadget_start()
2319 INIT_LIST_HEAD(&pdev->gadget.ep_list); in cdns2_gadget_start()
2320 pdev->eps_dma_pool = dma_pool_create("cdns2_eps_dma_pool", pdev->dev, in cdns2_gadget_start()
2322 if (!pdev->eps_dma_pool) { in cdns2_gadget_start()
2323 dev_err(pdev->dev, "Failed to create TRB dma pool\n"); in cdns2_gadget_start()
2324 ret = -ENOMEM; in cdns2_gadget_start()
2330 dev_err(pdev->dev, "Failed to create endpoints\n"); in cdns2_gadget_start()
2334 pdev->gadget.sg_supported = 1; in cdns2_gadget_start()
2336 pdev->zlp_buf = kzalloc(CDNS2_EP_ZLP_BUF_SIZE, GFP_KERNEL); in cdns2_gadget_start()
2337 if (!pdev->zlp_buf) { in cdns2_gadget_start()
2338 ret = -ENOMEM; in cdns2_gadget_start()
2343 buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma, in cdns2_gadget_start()
2345 pdev->ep0_preq.request.buf = buf; in cdns2_gadget_start()
2347 if (!pdev->ep0_preq.request.buf) { in cdns2_gadget_start()
2348 ret = -ENOMEM; in cdns2_gadget_start()
2353 ret = usb_add_gadget(&pdev->gadget); in cdns2_gadget_start()
2355 dev_err(pdev->dev, "Failed to add gadget\n"); in cdns2_gadget_start()
2362 dma_free_coherent(pdev->dev, 8, pdev->ep0_preq.request.buf, in cdns2_gadget_start()
2363 pdev->ep0_preq.request.dma); in cdns2_gadget_start()
2365 kfree(pdev->zlp_buf); in cdns2_gadget_start()
2367 dma_pool_destroy(pdev->eps_dma_pool); in cdns2_gadget_start()
2369 usb_put_gadget(&pdev->gadget); in cdns2_gadget_start()
2380 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_suspend()
2381 pdev->gadget.speed = USB_SPEED_UNKNOWN; in cdns2_gadget_suspend()
2384 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED); in cdns2_gadget_suspend()
2388 writeb(0, &pdev->interrupt_regs->usbien); in cdns2_gadget_suspend()
2389 writel(0, &pdev->adma_regs->ep_ien); in cdns2_gadget_suspend()
2390 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_suspend()
2399 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_resume()
2401 if (!pdev->gadget_driver) { in cdns2_gadget_resume()
2402 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_resume()
2409 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON); in cdns2_gadget_resume()
2411 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_resume()
2418 pm_runtime_mark_last_busy(pdev->dev); in cdns2_gadget_remove()
2419 pm_runtime_put_autosuspend(pdev->dev); in cdns2_gadget_remove()
2421 usb_del_gadget(&pdev->gadget); in cdns2_gadget_remove()
2424 dma_pool_destroy(pdev->eps_dma_pool); in cdns2_gadget_remove()
2425 kfree(pdev->zlp_buf); in cdns2_gadget_remove()
2426 usb_put_gadget(&pdev->gadget); in cdns2_gadget_remove()
2433 /* Ensure 32-bit DMA Mask. */ in cdns2_gadget_init()
2434 ret = dma_set_mask_and_coherent(pdev->dev, DMA_BIT_MASK(32)); in cdns2_gadget_init()
2436 dev_err(pdev->dev, "Failed to set dma mask: %d\n", ret); in cdns2_gadget_init()
2440 pm_runtime_get_sync(pdev->dev); in cdns2_gadget_init()
2446 pm_runtime_put_sync(pdev->dev); in cdns2_gadget_init()
2454 ret = devm_request_threaded_irq(pdev->dev, pdev->irq, in cdns2_gadget_init()
2458 dev_name(pdev->dev), in cdns2_gadget_init()