1 files changed, 140 insertions, 24 deletions

diff --git a/drivers/usb/musb/musb_cppi41.c b/drivers/usb/musb/musb_cppi41.c
index ff9d6de2b746..a12bd30401e0 100644
--- a/drivers/usb/musb/musb_cppi41.c
+++ b/drivers/usb/musb/musb_cppi41.c
@@ -38,6 +38,7 @@ struct cppi41_dma_channel {
 	u32 prog_len;
 	u32 transferred;
 	u32 packet_sz;
+	struct list_head tx_check;
 };
 
 #define MUSB_DMA_NUM_CHANNELS 15
@@ -47,6 +48,8 @@ struct cppi41_dma_controller {
 	struct cppi41_dma_channel rx_channel[MUSB_DMA_NUM_CHANNELS];
 	struct cppi41_dma_channel tx_channel[MUSB_DMA_NUM_CHANNELS];
 	struct musb *musb;
+	struct hrtimer early_tx;
+	struct list_head early_tx_list;
 	u32 rx_mode;
 	u32 tx_mode;
 	u32 auto_req;
@@ -96,31 +99,27 @@ static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
 	cppi41_channel->usb_toggle = toggle;
 }
 
-static void cppi41_dma_callback(void *private_data)
+static bool musb_is_tx_fifo_empty(struct musb_hw_ep *hw_ep)
 {
-	struct dma_channel *channel = private_data;
-	struct cppi41_dma_channel *cppi41_channel = channel->private_data;
-	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
-	struct musb *musb = hw_ep->musb;
-	unsigned long flags;
-	struct dma_tx_state txstate;
-	u32 transferred;
+	u8		epnum = hw_ep->epnum;
+	struct musb	*musb = hw_ep->musb;
+	void __iomem	*epio = musb->endpoints[epnum].regs;
+	u16		csr;
 
-	spin_lock_irqsave(&musb->lock, flags);
+	csr = musb_readw(epio, MUSB_TXCSR);
+	if (csr & MUSB_TXCSR_TXPKTRDY)
+		return false;
+	return true;
+}
 
-	dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
-			&txstate);
-	transferred = cppi41_channel->prog_len - txstate.residue;
-	cppi41_channel->transferred += transferred;
+static void cppi41_dma_callback(void *private_data);
 
-	dev_dbg(musb->controller, "DMA transfer done on hw_ep=%d bytes=%d/%d\n",
-		hw_ep->epnum, cppi41_channel->transferred,
-		cppi41_channel->total_len);
+static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel)
+{
+	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
+	struct musb *musb = hw_ep->musb;
 
-	update_rx_toggle(cppi41_channel);
-
-	if (cppi41_channel->transferred == cppi41_channel->total_len ||
-			transferred < cppi41_channel->packet_sz) {
+	if (!cppi41_channel->prog_len) {
 
 		/* done, complete */
 		cppi41_channel->channel.actual_len =
@@ -150,13 +149,11 @@ static void cppi41_dma_callback(void *private_data)
 				remain_bytes,
 				direction,
 				DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
-		if (WARN_ON(!dma_desc)) {
-			spin_unlock_irqrestore(&musb->lock, flags);
+		if (WARN_ON(!dma_desc))
 			return;
-		}
 
 		dma_desc->callback = cppi41_dma_callback;
-		dma_desc->callback_param = channel;
+		dma_desc->callback_param = &cppi41_channel->channel;
 		cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
 		dma_async_issue_pending(dc);
 
@@ -166,6 +163,117 @@ static void cppi41_dma_callback(void *private_data)
 			musb_writew(epio, MUSB_RXCSR, csr);
 		}
 	}
+}
+
+static enum hrtimer_restart cppi41_recheck_tx_req(struct hrtimer *timer)
+{
+	struct cppi41_dma_controller *controller;
+	struct cppi41_dma_channel *cppi41_channel, *n;
+	struct musb *musb;
+	unsigned long flags;
+	enum hrtimer_restart ret = HRTIMER_NORESTART;
+
+	controller = container_of(timer, struct cppi41_dma_controller,
+			early_tx);
+	musb = controller->musb;
+
+	spin_lock_irqsave(&musb->lock, flags);
+	list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list,
+			tx_check) {
+		bool empty;
+		struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
+
+		empty = musb_is_tx_fifo_empty(hw_ep);
+		if (empty) {
+			list_del_init(&cppi41_channel->tx_check);
+			cppi41_trans_done(cppi41_channel);
+		}
+	}
+
+	if (!list_empty(&controller->early_tx_list)) {
+		ret = HRTIMER_RESTART;
+		hrtimer_forward_now(&controller->early_tx,
+				ktime_set(0, 150 * NSEC_PER_USEC));
+	}
+
+	spin_unlock_irqrestore(&musb->lock, flags);
+	return ret;
+}
+
+static void cppi41_dma_callback(void *private_data)
+{
+	struct dma_channel *channel = private_data;
+	struct cppi41_dma_channel *cppi41_channel = channel->private_data;
+	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
+	struct musb *musb = hw_ep->musb;
+	unsigned long flags;
+	struct dma_tx_state txstate;
+	u32 transferred;
+	bool empty;
+
+	spin_lock_irqsave(&musb->lock, flags);
+
+	dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
+			&txstate);
+	transferred = cppi41_channel->prog_len - txstate.residue;
+	cppi41_channel->transferred += transferred;
+
+	dev_dbg(musb->controller, "DMA transfer done on hw_ep=%d bytes=%d/%d\n",
+		hw_ep->epnum, cppi41_channel->transferred,
+		cppi41_channel->total_len);
+
+	update_rx_toggle(cppi41_channel);
+
+	if (cppi41_channel->transferred == cppi41_channel->total_len ||
+			transferred < cppi41_channel->packet_sz)
+		cppi41_channel->prog_len = 0;
+
+	empty = musb_is_tx_fifo_empty(hw_ep);
+	if (empty) {
+		cppi41_trans_done(cppi41_channel);
+	} else {
+		struct cppi41_dma_controller *controller;
+		/*
+		 * On AM335x it has been observed that the TX interrupt fires
+		 * too early that means the TXFIFO is not yet empty but the DMA
+		 * engine says that it is done with the transfer. We don't
+		 * receive a FIFO empty interrupt so the only thing we can do is
+		 * to poll for the bit. On HS it usually takes 2us, on FS around
+		 * 110us - 150us depending on the transfer size.
+		 * We spin on HS (no longer than than 25us and setup a timer on
+		 * FS to check for the bit and complete the transfer.
+		 */
+		controller = cppi41_channel->controller;
+
+		if (musb->g.speed == USB_SPEED_HIGH) {
+			unsigned wait = 25;
+
+			do {
+				empty = musb_is_tx_fifo_empty(hw_ep);
+				if (empty)
+					break;
+				wait--;
+				if (!wait)
+					break;
+				udelay(1);
+			} while (1);
+
+			empty = musb_is_tx_fifo_empty(hw_ep);
+			if (empty) {
+				cppi41_trans_done(cppi41_channel);
+				goto out;
+			}
+		}
+		list_add_tail(&cppi41_channel->tx_check,
+				&controller->early_tx_list);
+		if (!hrtimer_active(&controller->early_tx)) {
+			hrtimer_start_range_ns(&controller->early_tx,
+				ktime_set(0, 140 * NSEC_PER_USEC),
+				40 * NSEC_PER_USEC,
+				HRTIMER_MODE_REL);
+		}
+	}
+out:
 	spin_unlock_irqrestore(&musb->lock, flags);
 }
 
@@ -364,6 +472,8 @@ static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket,
 		WARN_ON(1);
 		return 1;
 	}
+	if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK)
+		return 0;
 	if (cppi41_channel->is_tx)
 		return 1;
 	/* AM335x Advisory 1.0.13. No workaround for device RX mode */
@@ -388,6 +498,7 @@ static int cppi41_dma_channel_abort(struct dma_channel *channel)
 	if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)
 		return 0;
 
+	list_del_init(&cppi41_channel->tx_check);
 	if (is_tx) {
 		csr = musb_readw(epio, MUSB_TXCSR);
 		csr &= ~MUSB_TXCSR_DMAENAB;
@@ -495,6 +606,7 @@ static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller)
 		cppi41_channel->controller = controller;
 		cppi41_channel->port_num = port;
 		cppi41_channel->is_tx = is_tx;
+		INIT_LIST_HEAD(&cppi41_channel->tx_check);
 
 		musb_dma = &cppi41_channel->channel;
 		musb_dma->private_data = cppi41_channel;
@@ -520,6 +632,7 @@ void dma_controller_destroy(struct dma_controller *c)
 	struct cppi41_dma_controller *controller = container_of(c,
 			struct cppi41_dma_controller, controller);
 
+	hrtimer_cancel(&controller->early_tx);
 	cppi41_dma_controller_stop(controller);
 	kfree(controller);
 }
@@ -539,6 +652,9 @@ struct dma_controller *dma_controller_create(struct musb *musb,
 	if (!controller)
 		goto kzalloc_fail;
 
+	hrtimer_init(&controller->early_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	controller->early_tx.function = cppi41_recheck_tx_req;
+	INIT_LIST_HEAD(&controller->early_tx_list);
 	controller->musb = musb;
 
 	controller->controller.channel_alloc = cppi41_dma_channel_allocate;