usb: dwc3: convert TRBs into bitshifts

this will get rid of a useless memcpy on
IRQ handling, thus improving driver performance.

Tested with OMAP5430 running g_mass_storage on
SuperSpeed and HighSpeed.

Note that we are removing the little endian access
of the TRB and all accesses will be in System endianness,
if there happens to be a system in BE, bit 12 of GSBUSCFG0
should be set so that HW does byte invariant BE accesses
when fetching TRBs.

Signed-off-by: Felipe Balbi <balbi@ti.com>

3 files changed, 97 insertions, 162 deletions

diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 4dac9828577e..696b20a8b1fa 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -301,7 +301,7 @@
 
 /* Structures */
 
-struct dwc3_trb_hw;
+struct dwc3_trb;
 
 /**
  * struct dwc3_event_buffer - Software event buffer representation
@@ -356,7 +356,7 @@ struct dwc3_ep {
 	struct list_head	request_list;
 	struct list_head	req_queued;
 
-	struct dwc3_trb_hw	*trb_pool;
+	struct dwc3_trb		*trb_pool;
 	dma_addr_t		trb_pool_dma;
 	u32			free_slot;
 	u32			busy_slot;
@@ -431,102 +431,49 @@ enum dwc3_device_state {
 	DWC3_CONFIGURED_STATE,
 };
 
-/**
- * struct dwc3_trb - transfer request block
- * @bpl: lower 32bit of the buffer
- * @bph: higher 32bit of the buffer
- * @length: buffer size (up to 16mb - 1)
- * @pcm1: packet count m1
- * @trbsts: trb status
- *	0 = ok
- *	1 = missed isoc
- *	2 = setup pending
- * @hwo: hardware owner of descriptor
- * @lst: last trb
- * @chn: chain buffers
- * @csp: continue on short packets (only supported on isoc eps)
- * @trbctl: trb control
- *	1 = normal
- *	2 = control-setup
- *	3 = control-status-2
- *	4 = control-status-3
- *	5 = control-data (first trb of data stage)
- *	6 = isochronous-first (first trb of service interval)
- *	7 = isochronous
- *	8 = link trb
- *	others = reserved
- * @isp_imi: interrupt on short packet / interrupt on missed isoc
- * @ioc: interrupt on complete
- * @sid_sofn: Stream ID / SOF Number
- */
-struct dwc3_trb {
-	u64             bplh;
-
-	union {
-		struct {
-			u32             length:24;
-			u32             pcm1:2;
-			u32             reserved27_26:2;
-			u32             trbsts:4;
-#define DWC3_TRB_STS_OKAY                       0
-#define DWC3_TRB_STS_MISSED_ISOC                1
-#define DWC3_TRB_STS_SETUP_PENDING              2
-		};
-		u32 len_pcm;
-	};
-
-	union {
-		struct {
-			u32             hwo:1;
-			u32             lst:1;
-			u32             chn:1;
-			u32             csp:1;
-			u32             trbctl:6;
-			u32             isp_imi:1;
-			u32             ioc:1;
-			u32             reserved13_12:2;
-			u32             sid_sofn:16;
-			u32             reserved31_30:2;
-		};
-		u32 control;
-	};
-} __packed;
+/* TRB Length, PCM and Status */
+#define DWC3_TRB_SIZE_MASK	(0x00ffffff)
+#define DWC3_TRB_SIZE_LENGTH(n)	((n) & DWC3_TRB_SIZE_MASK)
+#define DWC3_TRB_SIZE_PCM1(n)	(((n) & 0x03) << 24)
+#define DWC3_TRB_SIZE_TRBSTS(n)	(((n) & (0x0f << 28) >> 28))
+
+#define DWC3_TRBSTS_OK			0
+#define DWC3_TRBSTS_MISSED_ISOC		1
+#define DWC3_TRBSTS_SETUP_PENDING	2
+
+/* TRB Control */
+#define DWC3_TRB_CTRL_HWO		(1 << 0)
+#define DWC3_TRB_CTRL_LST		(1 << 1)
+#define DWC3_TRB_CTRL_CHN		(1 << 2)
+#define DWC3_TRB_CTRL_CSP		(1 << 3)
+#define DWC3_TRB_CTRL_TRBCTL(n)		(((n) & 0x3f) << 4)
+#define DWC3_TRB_CTRL_ISP_IMI		(1 << 10)
+#define DWC3_TRB_CTRL_IOC		(1 << 11)
+#define DWC3_TRB_CTRL_SID_SOFN(n)	(((n) & 0xffff) << 14)
+
+#define DWC3_TRBCTL_NORMAL		DWC3_TRB_CTRL_TRBCTL(1)
+#define DWC3_TRBCTL_CONTROL_SETUP	DWC3_TRB_CTRL_TRBCTL(2)
+#define DWC3_TRBCTL_CONTROL_STATUS2	DWC3_TRB_CTRL_TRBCTL(3)
+#define DWC3_TRBCTL_CONTROL_STATUS3	DWC3_TRB_CTRL_TRBCTL(4)
+#define DWC3_TRBCTL_CONTROL_DATA	DWC3_TRB_CTRL_TRBCTL(5)
+#define DWC3_TRBCTL_ISOCHRONOUS_FIRST	DWC3_TRB_CTRL_TRBCTL(6)
+#define DWC3_TRBCTL_ISOCHRONOUS		DWC3_TRB_CTRL_TRBCTL(7)
+#define DWC3_TRBCTL_LINK_TRB		DWC3_TRB_CTRL_TRBCTL(8)
 
 /**
- * struct dwc3_trb_hw - transfer request block (hw format)
+ * struct dwc3_trb - transfer request block (hw format)
  * @bpl: DW0-3
  * @bph: DW4-7
  * @size: DW8-B
  * @trl: DWC-F
  */
-struct dwc3_trb_hw {
-	__le32		bpl;
-	__le32		bph;
-	__le32		size;
-	__le32		ctrl;
+struct dwc3_trb {
+	u32		bpl;
+	u32		bph;
+	u32		size;
+	u32		ctrl;
 } __packed;
 
-static inline void dwc3_trb_to_hw(struct dwc3_trb *nat, struct dwc3_trb_hw *hw)
-{
-	hw->bpl = cpu_to_le32(lower_32_bits(nat->bplh));
-	hw->bph = cpu_to_le32(upper_32_bits(nat->bplh));
-	hw->size = cpu_to_le32p(&nat->len_pcm);
-	/* HWO is written last */
-	hw->ctrl = cpu_to_le32p(&nat->control);
-}
-
-static inline void dwc3_trb_to_nat(struct dwc3_trb_hw *hw, struct dwc3_trb *nat)
-{
-	u64 bplh;
-
-	bplh = le32_to_cpup(&hw->bpl);
-	bplh |= (u64) le32_to_cpup(&hw->bph) << 32;
-	nat->bplh = bplh;
-
-	nat->len_pcm = le32_to_cpup(&hw->size);
-	nat->control = le32_to_cpup(&hw->ctrl);
-}
-
 /**
  * dwc3_hwparams - copy of HWPARAMS registers
  * @hwparams0 - GHWPARAMS0
@@ -573,7 +520,7 @@ struct dwc3_request {
 	struct dwc3_ep		*dep;
 
 	u8			epnum;
-	struct dwc3_trb_hw	*trb;
+	struct dwc3_trb		*trb;
 	dma_addr_t		trb_dma;
 
 	unsigned		direction:1;
@@ -624,7 +571,7 @@ struct dwc3_request {
  */
 struct dwc3 {
 	struct usb_ctrlrequest	*ctrl_req;
-	struct dwc3_trb_hw	*ep0_trb;
+	struct dwc3_trb		*ep0_trb;
 	void			*ep0_bounce;
 	u8			*setup_buf;
 	dma_addr_t		ctrl_req_addr;
@@ -691,19 +638,6 @@ struct dwc3 {
 
 /* -------------------------------------------------------------------------- */
 
-#define DWC3_TRBSTS_OK			0
-#define DWC3_TRBSTS_MISSED_ISOC		1
-#define DWC3_TRBSTS_SETUP_PENDING	2
-
-#define DWC3_TRBCTL_NORMAL		1
-#define DWC3_TRBCTL_CONTROL_SETUP	2
-#define DWC3_TRBCTL_CONTROL_STATUS2	3
-#define DWC3_TRBCTL_CONTROL_STATUS3	4
-#define DWC3_TRBCTL_CONTROL_DATA	5
-#define DWC3_TRBCTL_ISOCHRONOUS_FIRST	6
-#define DWC3_TRBCTL_ISOCHRONOUS		7
-#define DWC3_TRBCTL_LINK_TRB		8
-
 /* -------------------------------------------------------------------------- */
 
 struct dwc3_event_type {
diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c
index c20e30c8b695..5a067090f27e 100644
--- a/drivers/usb/dwc3/ep0.c
+++ b/drivers/usb/dwc3/ep0.c
@@ -76,8 +76,7 @@ static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum, dma_addr_t buf_dma,
 		u32 len, u32 type)
 {
 	struct dwc3_gadget_ep_cmd_params params;
-	struct dwc3_trb_hw		*trb_hw;
-	struct dwc3_trb			trb;
+	struct dwc3_trb			*trb;
 	struct dwc3_ep			*dep;
 
 	int				ret;
@@ -88,19 +87,17 @@ static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum, dma_addr_t buf_dma,
 		return 0;
 	}
 
-	trb_hw = dwc->ep0_trb;
-	memset(&trb, 0, sizeof(trb));
+	trb = dwc->ep0_trb;
 
-	trb.trbctl = type;
-	trb.bplh = buf_dma;
-	trb.length = len;
+	trb->bpl = lower_32_bits(buf_dma);
+	trb->bph = upper_32_bits(buf_dma);
+	trb->size = len;
+	trb->ctrl = type;
 
-	trb.hwo	= 1;
-	trb.lst	= 1;
-	trb.ioc	= 1;
-	trb.isp_imi = 1;
-
-	dwc3_trb_to_hw(&trb, trb_hw);
+	trb->ctrl |= (DWC3_TRB_CTRL_HWO
+			| DWC3_TRB_CTRL_LST
+			| DWC3_TRB_CTRL_IOC
+			| DWC3_TRB_CTRL_ISP_IMI);
 
 	memset(&params, 0, sizeof(params));
 	params.param0 = upper_32_bits(dwc->ep0_trb_addr);
@@ -544,9 +541,10 @@ static void dwc3_ep0_complete_data(struct dwc3 *dwc,
 {
 	struct dwc3_request	*r = NULL;
 	struct usb_request	*ur;
-	struct dwc3_trb		trb;
+	struct dwc3_trb		*trb;
 	struct dwc3_ep		*ep0;
 	u32			transferred;
+	u32			length;
 	u8			epnum;
 
 	epnum = event->endpoint_number;
@@ -557,16 +555,16 @@ static void dwc3_ep0_complete_data(struct dwc3 *dwc,
 	r = next_request(&ep0->request_list);
 	ur = &r->request;
 
-	dwc3_trb_to_nat(dwc->ep0_trb, &trb);
+	trb = dwc->ep0_trb;
+	length = trb->size & DWC3_TRB_SIZE_MASK;
 
 	if (dwc->ep0_bounced) {
-
 		transferred = min_t(u32, ur->length,
-				ep0->endpoint.maxpacket - trb.length);
+				ep0->endpoint.maxpacket - length);
 		memcpy(ur->buf, dwc->ep0_bounce, transferred);
 		dwc->ep0_bounced = false;
 	} else {
-		transferred = ur->length - trb.length;
+		transferred = ur->length - length;
 		ur->actual += transferred;
 	}
 
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 76327005d54c..eeba2227c28f 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -378,7 +378,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
 }
 
 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
-		struct dwc3_trb_hw *trb)
+		struct dwc3_trb *trb)
 {
 	u32		offset = (char *) trb - (char *) dep->trb_pool;
 
@@ -527,9 +527,8 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 		return ret;
 
 	if (!(dep->flags & DWC3_EP_ENABLED)) {
-		struct dwc3_trb_hw	*trb_st_hw;
-		struct dwc3_trb_hw	*trb_link_hw;
-		struct dwc3_trb		trb_link;
+		struct dwc3_trb	*trb_st_hw;
+		struct dwc3_trb	*trb_link;
 
 		ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 		if (ret)
@@ -552,12 +551,12 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 		/* Link TRB for ISOC. The HWO but is never reset */
 		trb_st_hw = &dep->trb_pool[0];
 
-		trb_link.bplh = dwc3_trb_dma_offset(dep, trb_st_hw);
-		trb_link.trbctl = DWC3_TRBCTL_LINK_TRB;
-		trb_link.hwo = true;
+		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 
-		trb_link_hw = &dep->trb_pool[DWC3_TRB_NUM - 1];
-		dwc3_trb_to_hw(&trb_link, trb_link_hw);
+		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
+		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
+		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
+		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 	}
 
 	return 0;
@@ -744,8 +743,7 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 		unsigned length, unsigned last, unsigned chain)
 {
 	struct dwc3		*dwc = dep->dwc;
-	struct dwc3_trb_hw	*trb_hw;
-	struct dwc3_trb		trb;
+	struct dwc3_trb		*trb;
 
 	unsigned int		cur_slot;
 
@@ -754,7 +752,7 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 			length, last ? " last" : "",
 			chain ? " chain" : "");
 
-	trb_hw = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
+	trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
 	cur_slot = dep->free_slot;
 	dep->free_slot++;
 
@@ -763,40 +761,32 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 			usb_endpoint_xfer_isoc(dep->desc))
 		return;
 
-	memset(&trb, 0, sizeof(trb));
 	if (!req->trb) {
 		dwc3_gadget_move_request_queued(req);
-		req->trb = trb_hw;
-		req->trb_dma = dwc3_trb_dma_offset(dep, trb_hw);
+		req->trb = trb;
+		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
 	}
 
-	if (usb_endpoint_xfer_isoc(dep->desc)) {
-		trb.isp_imi = true;
-		trb.csp = true;
-	} else {
-		trb.chn = chain;
-		trb.lst = last;
-	}
-
-	if (usb_endpoint_xfer_bulk(dep->desc) && dep->stream_capable)
-		trb.sid_sofn = req->request.stream_id;
+	trb->size = DWC3_TRB_SIZE_LENGTH(length);
+	trb->bpl = lower_32_bits(dma);
+	trb->bph = upper_32_bits(dma);
 
 	switch (usb_endpoint_type(dep->desc)) {
 	case USB_ENDPOINT_XFER_CONTROL:
-		trb.trbctl = DWC3_TRBCTL_CONTROL_SETUP;
+		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 		break;
 
 	case USB_ENDPOINT_XFER_ISOC:
-		trb.trbctl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
+		trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 
 		/* IOC every DWC3_TRB_NUM / 4 so we can refill */
 		if (!(cur_slot % (DWC3_TRB_NUM / 4)))
-			trb.ioc = last;
+			trb->ctrl |= DWC3_TRB_CTRL_IOC;
 		break;
 
 	case USB_ENDPOINT_XFER_BULK:
 	case USB_ENDPOINT_XFER_INT:
-		trb.trbctl = DWC3_TRBCTL_NORMAL;
+		trb->ctrl = DWC3_TRBCTL_NORMAL;
 		break;
 	default:
 		/*
@@ -806,11 +796,21 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 		BUG();
 	}
 
-	trb.length	= length;
-	trb.bplh	= dma;
-	trb.hwo		= true;
+	if (usb_endpoint_xfer_isoc(dep->desc)) {
+		trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
+		trb->ctrl |= DWC3_TRB_CTRL_CSP;
+	} else {
+		if (chain)
+			trb->ctrl |= DWC3_TRB_CTRL_CHN;
+
+		if (last)
+			trb->ctrl |= DWC3_TRB_CTRL_LST;
+	}
+
+	if (usb_endpoint_xfer_bulk(dep->desc) && dep->stream_capable)
+		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
 
-	dwc3_trb_to_hw(&trb, trb_hw);
+	trb->ctrl |= DWC3_TRB_CTRL_HWO;
 }
 
 /*
@@ -1542,7 +1542,7 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 		const struct dwc3_event_depevt *event, int status)
 {
 	struct dwc3_request	*req;
-	struct dwc3_trb         trb;
+	struct dwc3_trb		*trb;
 	unsigned int		count;
 	unsigned int		s_pkt = 0;
 
@@ -1553,9 +1553,9 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 			return 1;
 		}
 
-		dwc3_trb_to_nat(req->trb, &trb);
+		trb = req->trb;
 
-		if (trb.hwo && status != -ESHUTDOWN)
+		if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
 			/*
 			 * We continue despite the error. There is not much we
 			 * can do. If we don't clean in up we loop for ever. If
@@ -1566,7 +1566,7 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 			 */
 			dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
 					dep->name, req->trb);
-		count = trb.length;
+		count = trb->size & DWC3_TRB_SIZE_MASK;
 
 		if (dep->direction) {
 			if (count) {
@@ -1590,13 +1590,16 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 		dwc3_gadget_giveback(dep, req, status);
 		if (s_pkt)
 			break;
-		if ((event->status & DEPEVT_STATUS_LST) && trb.lst)
+		if ((event->status & DEPEVT_STATUS_LST) &&
+				(trb->ctrl & DWC3_TRB_CTRL_LST))
 			break;
-		if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
+		if ((event->status & DEPEVT_STATUS_IOC) &&
+				(trb->ctrl & DWC3_TRB_CTRL_IOC))
 			break;
 	} while (1);
 
-	if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
+	if ((event->status & DEPEVT_STATUS_IOC) &&
+			(trb->ctrl & DWC3_TRB_CTRL_IOC))
 		return 0;
 	return 1;
 }