diff options
Diffstat (limited to 'drivers/dma')
| -rw-r--r-- | drivers/dma/dw/Kconfig | 7 | ||||
| -rw-r--r-- | drivers/dma/dw/core.c | 332 | ||||
| -rw-r--r-- | drivers/dma/dw/regs.h | 50 | ||||
| -rw-r--r-- | drivers/dma/ep93xx_dma.c | 39 | ||||
| -rw-r--r-- | drivers/dma/ioat/init.c | 24 | ||||
| -rw-r--r-- | drivers/dma/mv_xor_v2.c | 109 | ||||
| -rw-r--r-- | drivers/dma/omap-dma.c | 39 | ||||
| -rw-r--r-- | drivers/dma/pl330.c | 3 | ||||
| -rw-r--r-- | drivers/dma/sh/rcar-dmac.c | 3 | ||||
| -rw-r--r-- | drivers/dma/sh/usb-dmac.c | 2 |
10 files changed, 121 insertions, 487 deletions
diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig index 5a37b9fcf40d..04b9728c1d26 100644 --- a/drivers/dma/dw/Kconfig +++ b/drivers/dma/dw/Kconfig @@ -6,17 +6,12 @@ config DW_DMAC_CORE tristate select DMA_ENGINE -config DW_DMAC_BIG_ENDIAN_IO - bool - config DW_DMAC tristate "Synopsys DesignWare AHB DMA platform driver" select DW_DMAC_CORE - select DW_DMAC_BIG_ENDIAN_IO if AVR32 - default y if CPU_AT32AP7000 help Support the Synopsys DesignWare AHB DMA controller. This - can be integrated in chips such as the Atmel AT32ap7000. + can be integrated in chips such as the Intel Cherrytrail. config DW_DMAC_PCI tristate "Synopsys DesignWare AHB DMA PCI driver" diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index e500950dad82..f43e6dafe446 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -561,92 +561,14 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc) dwc_descriptor_complete(dwc, bad_desc, true); } -/* --------------------- Cyclic DMA API extensions -------------------- */ - -dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan) -{ - struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - return channel_readl(dwc, SAR); -} -EXPORT_SYMBOL(dw_dma_get_src_addr); - -dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan) -{ - struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - return channel_readl(dwc, DAR); -} -EXPORT_SYMBOL(dw_dma_get_dst_addr); - -/* Called with dwc->lock held and all DMAC interrupts disabled */ -static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc, - u32 status_block, u32 status_err, u32 status_xfer) -{ - unsigned long flags; - - if (status_block & dwc->mask) { - void (*callback)(void *param); - void *callback_param; - - dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n", - channel_readl(dwc, LLP)); - dma_writel(dw, CLEAR.BLOCK, dwc->mask); - - callback = dwc->cdesc->period_callback; - callback_param = dwc->cdesc->period_callback_param; - - if (callback) - callback(callback_param); - } - - /* - * Error and transfer complete are highly unlikely, and will most - * likely be due to a configuration error by the user. - */ - if (unlikely(status_err & dwc->mask) || - unlikely(status_xfer & dwc->mask)) { - unsigned int i; - - dev_err(chan2dev(&dwc->chan), - "cyclic DMA unexpected %s interrupt, stopping DMA transfer\n", - status_xfer ? "xfer" : "error"); - - spin_lock_irqsave(&dwc->lock, flags); - - dwc_dump_chan_regs(dwc); - - dwc_chan_disable(dw, dwc); - - /* Make sure DMA does not restart by loading a new list */ - channel_writel(dwc, LLP, 0); - channel_writel(dwc, CTL_LO, 0); - channel_writel(dwc, CTL_HI, 0); - - dma_writel(dw, CLEAR.BLOCK, dwc->mask); - dma_writel(dw, CLEAR.ERROR, dwc->mask); - dma_writel(dw, CLEAR.XFER, dwc->mask); - - for (i = 0; i < dwc->cdesc->periods; i++) - dwc_dump_lli(dwc, dwc->cdesc->desc[i]); - - spin_unlock_irqrestore(&dwc->lock, flags); - } - - /* Re-enable interrupts */ - channel_set_bit(dw, MASK.BLOCK, dwc->mask); -} - -/* ------------------------------------------------------------------------- */ - static void dw_dma_tasklet(unsigned long data) { struct dw_dma *dw = (struct dw_dma *)data; struct dw_dma_chan *dwc; - u32 status_block; u32 status_xfer; u32 status_err; unsigned int i; - status_block = dma_readl(dw, RAW.BLOCK); status_xfer = dma_readl(dw, RAW.XFER); status_err = dma_readl(dw, RAW.ERROR); @@ -655,8 +577,7 @@ static void dw_dma_tasklet(unsigned long data) for (i = 0; i < dw->dma.chancnt; i++) { dwc = &dw->chan[i]; if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) - dwc_handle_cyclic(dw, dwc, status_block, status_err, - status_xfer); + dev_vdbg(dw->dma.dev, "Cyclic xfer is not implemented\n"); else if (status_err & (1 << i)) dwc_handle_error(dw, dwc); else if (status_xfer & (1 << i)) @@ -1264,255 +1185,6 @@ static void dwc_free_chan_resources(struct dma_chan *chan) dev_vdbg(chan2dev(chan), "%s: done\n", __func__); } -/* --------------------- Cyclic DMA API extensions -------------------- */ - -/** - * dw_dma_cyclic_start - start the cyclic DMA transfer - * @chan: the DMA channel to start - * - * Must be called with soft interrupts disabled. Returns zero on success or - * -errno on failure. - */ -int dw_dma_cyclic_start(struct dma_chan *chan) -{ - struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - struct dw_dma *dw = to_dw_dma(chan->device); - unsigned long flags; - - if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) { - dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n"); - return -ENODEV; - } - - spin_lock_irqsave(&dwc->lock, flags); - - /* Enable interrupts to perform cyclic transfer */ - channel_set_bit(dw, MASK.BLOCK, dwc->mask); - - dwc_dostart(dwc, dwc->cdesc->desc[0]); - - spin_unlock_irqrestore(&dwc->lock, flags); - - return 0; -} -EXPORT_SYMBOL(dw_dma_cyclic_start); - -/** - * dw_dma_cyclic_stop - stop the cyclic DMA transfer - * @chan: the DMA channel to stop - * - * Must be called with soft interrupts disabled. - */ -void dw_dma_cyclic_stop(struct dma_chan *chan) -{ - struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - struct dw_dma *dw = to_dw_dma(dwc->chan.device); - unsigned long flags; - - spin_lock_irqsave(&dwc->lock, flags); - - dwc_chan_disable(dw, dwc); - - spin_unlock_irqrestore(&dwc->lock, flags); -} -EXPORT_SYMBOL(dw_dma_cyclic_stop); - -/** - * dw_dma_cyclic_prep - prepare the cyclic DMA transfer - * @chan: the DMA channel to prepare - * @buf_addr: physical DMA address where the buffer starts - * @buf_len: total number of bytes for the entire buffer - * @period_len: number of bytes for each period - * @direction: transfer direction, to or from device - * - * Must be called before trying to start the transfer. Returns a valid struct - * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful. - */ -struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan, - dma_addr_t buf_addr, size_t buf_len, size_t period_len, - enum dma_transfer_direction direction) -{ - struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - struct dma_slave_config *sconfig = &dwc->dma_sconfig; - struct dw_cyclic_desc *cdesc; - struct dw_cyclic_desc *retval = NULL; - struct dw_desc *desc; - struct dw_desc *last = NULL; - u8 lms = DWC_LLP_LMS(dwc->dws.m_master); - unsigned long was_cyclic; - unsigned int reg_width; - unsigned int periods; - unsigned int i; - unsigned long flags; - - spin_lock_irqsave(&dwc->lock, flags); - if (dwc->nollp) { - spin_unlock_irqrestore(&dwc->lock, flags); - dev_dbg(chan2dev(&dwc->chan), - "channel doesn't support LLP transfers\n"); - return ERR_PTR(-EINVAL); - } - - if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) { - spin_unlock_irqrestore(&dwc->lock, flags); - dev_dbg(chan2dev(&dwc->chan), - "queue and/or active list are not empty\n"); - return ERR_PTR(-EBUSY); - } - - was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags); - spin_unlock_irqrestore(&dwc->lock, flags); - if (was_cyclic) { - dev_dbg(chan2dev(&dwc->chan), - "channel already prepared for cyclic DMA\n"); - return ERR_PTR(-EBUSY); - } - - retval = ERR_PTR(-EINVAL); - - if (unlikely(!is_slave_direction(direction))) - goto out_err; - - dwc->direction = direction; - - if (direction == DMA_MEM_TO_DEV) - reg_width = __ffs(sconfig->dst_addr_width); - else - reg_width = __ffs(sconfig->src_addr_width); - - periods = buf_len / period_len; - - /* Check for too big/unaligned periods and unaligned DMA buffer. */ - if (period_len > (dwc->block_size << reg_width)) - goto out_err; - if (unlikely(period_len & ((1 << reg_width) - 1))) - goto out_err; - if (unlikely(buf_addr & ((1 << reg_width) - 1))) - goto out_err; - - retval = ERR_PTR(-ENOMEM); - - cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL); - if (!cdesc) - goto out_err; - - cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL); - if (!cdesc->desc) - goto out_err_alloc; - - for (i = 0; i < periods; i++) { - desc = dwc_desc_get(dwc); - if (!desc) - goto out_err_desc_get; - - switch (direction) { - case DMA_MEM_TO_DEV: - lli_write(desc, dar, sconfig->dst_addr); - lli_write(desc, sar, buf_addr + period_len * i); - lli_write(desc, ctllo, (DWC_DEFAULT_CTLLO(chan) - | DWC_CTLL_DST_WIDTH(reg_width) - | DWC_CTLL_SRC_WIDTH(reg_width) - | DWC_CTLL_DST_FIX - | DWC_CTLL_SRC_INC - | DWC_CTLL_INT_EN)); - - lli_set(desc, ctllo, sconfig->device_fc ? - DWC_CTLL_FC(DW_DMA_FC_P_M2P) : - DWC_CTLL_FC(DW_DMA_FC_D_M2P)); - - break; - case DMA_DEV_TO_MEM: - lli_write(desc, dar, buf_addr + period_len * i); - lli_write(desc, sar, sconfig->src_addr); - lli_write(desc, ctllo, (DWC_DEFAULT_CTLLO(chan) - | DWC_CTLL_SRC_WIDTH(reg_width) - | DWC_CTLL_DST_WIDTH(reg_width) - | DWC_CTLL_DST_INC - | DWC_CTLL_SRC_FIX - | DWC_CTLL_INT_EN)); - - lli_set(desc, ctllo, sconfig->device_fc ? - DWC_CTLL_FC(DW_DMA_FC_P_P2M) : - DWC_CTLL_FC(DW_DMA_FC_D_P2M)); - - break; - default: - break; - } - - lli_write(desc, ctlhi, period_len >> reg_width); - cdesc->desc[i] = desc; - - if (last) - lli_write(last, llp, desc->txd.phys | lms); - - last = desc; - } - - /* Let's make a cyclic list */ - lli_write(last, llp, cdesc->desc[0]->txd.phys | lms); - - dev_dbg(chan2dev(&dwc->chan), - "cyclic prepared buf %pad len %zu period %zu periods %d\n", - &buf_addr, buf_len, period_len, periods); - - cdesc->periods = periods; - dwc->cdesc = cdesc; - - return cdesc; - -out_err_desc_get: - while (i--) - dwc_desc_put(dwc, cdesc->desc[i]); -out_err_alloc: - kfree(cdesc); -out_err: - clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags); - return (struct dw_cyclic_desc *)retval; -} -EXPORT_SYMBOL(dw_dma_cyclic_prep); - -/** - * dw_dma_cyclic_free - free a prepared cyclic DMA transfer - * @chan: the DMA channel to free - */ -void dw_dma_cyclic_free(struct dma_chan *chan) -{ - struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - struct dw_dma *dw = to_dw_dma(dwc->chan.device); - struct dw_cyclic_desc *cdesc = dwc->cdesc; - unsigned int i; - unsigned long flags; - - dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__); - - if (!cdesc) - return; - - spin_lock_irqsave(&dwc->lock, flags); - - dwc_chan_disable(dw, dwc); - - dma_writel(dw, CLEAR.BLOCK, dwc->mask); - dma_writel(dw, CLEAR.ERROR, dwc->mask); - dma_writel(dw, CLEAR.XFER, dwc->mask); - - spin_unlock_irqrestore(&dwc->lock, flags); - - for (i = 0; i < cdesc->periods; i++) - dwc_desc_put(dwc, cdesc->desc[i]); - - kfree(cdesc->desc); - kfree(cdesc); - - dwc->cdesc = NULL; - - clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags); -} -EXPORT_SYMBOL(dw_dma_cyclic_free); - -/*----------------------------------------------------------------------*/ - int dw_dma_probe(struct dw_dma_chip *chip) { struct dw_dma_platform_data *pdata; @@ -1642,7 +1314,7 @@ int dw_dma_probe(struct dw_dma_chip *chip) if (autocfg) { unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1; void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r]; - unsigned int dwc_params = dma_readl_native(addr); + unsigned int dwc_params = readl(addr); dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i, dwc_params); diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h index 32a328721c88..09e7dfdbb790 100644 --- a/drivers/dma/dw/regs.h +++ b/drivers/dma/dw/regs.h @@ -116,20 +116,6 @@ struct dw_dma_regs { DW_REG(GLOBAL_CFG); }; -/* - * Big endian I/O access when reading and writing to the DMA controller - * registers. This is needed on some platforms, like the Atmel AVR32 - * architecture. - */ - -#ifdef CONFIG_DW_DMAC_BIG_ENDIAN_IO -#define dma_readl_native ioread32be -#define dma_writel_native iowrite32be -#else -#define dma_readl_native readl -#define dma_writel_native writel -#endif - /* Bitfields in DW_PARAMS */ #define DW_PARAMS_NR_CHAN 8 /* number of channels */ #define DW_PARAMS_NR_MASTER 11 /* number of AHB masters */ @@ -280,7 +266,6 @@ struct dw_dma_chan { unsigned long flags; struct list_head active_list; struct list_head queue; - struct dw_cyclic_desc *cdesc; unsigned int descs_allocated; @@ -302,9 +287,9 @@ __dwc_regs(struct dw_dma_chan *dwc) } #define channel_readl(dwc, name) \ - dma_readl_native(&(__dwc_regs(dwc)->name)) + readl(&(__dwc_regs(dwc)->name)) #define channel_writel(dwc, name, val) \ - dma_writel_native((val), &(__dwc_regs(dwc)->name)) + writel((val), &(__dwc_regs(dwc)->name)) static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan) { @@ -333,9 +318,9 @@ static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw) } #define dma_readl(dw, name) \ - dma_readl_native(&(__dw_regs(dw)->name)) + readl(&(__dw_regs(dw)->name)) #define dma_writel(dw, name, val) \ - dma_writel_native((val), &(__dw_regs(dw)->name)) + writel((val), &(__dw_regs(dw)->name)) #define idma32_readq(dw, name) \ hi_lo_readq(&(__dw_regs(dw)->name)) @@ -352,43 +337,30 @@ static inline struct dw_dma *to_dw_dma(struct dma_device *ddev) return container_of(ddev, struct dw_dma, dma); } -#ifdef CONFIG_DW_DMAC_BIG_ENDIAN_IO -typedef __be32 __dw32; -#else -typedef __le32 __dw32; -#endif - /* LLI == Linked List Item; a.k.a. DMA block descriptor */ struct dw_lli { /* values that are not changed by hardware */ - __dw32 sar; - __dw32 dar; - __dw32 llp; /* chain to next lli */ - __dw32 ctllo; + __le32 sar; + __le32 dar; + __le32 llp; /* chain to next lli */ + __le32 ctllo; /* values that may get written back: */ - __dw32 ctlhi; + __le32 ctlhi; /* sstat and dstat can snapshot peripheral register state. * silicon config may discard either or both... */ - __dw32 sstat; - __dw32 dstat; + __le32 sstat; + __le32 dstat; }; struct dw_desc { /* FIRST values the hardware uses */ struct dw_lli lli; -#ifdef CONFIG_DW_DMAC_BIG_ENDIAN_IO -#define lli_set(d, reg, v) ((d)->lli.reg |= cpu_to_be32(v)) -#define lli_clear(d, reg, v) ((d)->lli.reg &= ~cpu_to_be32(v)) -#define lli_read(d, reg) be32_to_cpu((d)->lli.reg) -#define lli_write(d, reg, v) ((d)->lli.reg = cpu_to_be32(v)) -#else #define lli_set(d, reg, v) ((d)->lli.reg |= cpu_to_le32(v)) #define lli_clear(d, reg, v) ((d)->lli.reg &= ~cpu_to_le32(v)) #define lli_read(d, reg) le32_to_cpu((d)->lli.reg) #define lli_write(d, reg, v) ((d)->lli.reg = cpu_to_le32(v)) -#endif /* THEN values for driver housekeeping */ struct list_head desc_node; diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index d37e8dda8079..ec240592f5c8 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -201,6 +201,7 @@ struct ep93xx_dma_engine { struct dma_device dma_dev; bool m2m; int (*hw_setup)(struct ep93xx_dma_chan *); + void (*hw_synchronize)(struct ep93xx_dma_chan *); void (*hw_shutdown)(struct ep93xx_dma_chan *); void (*hw_submit)(struct ep93xx_dma_chan *); int (*hw_interrupt)(struct ep93xx_dma_chan *); @@ -323,6 +324,8 @@ static int m2p_hw_setup(struct ep93xx_dma_chan *edmac) | M2P_CONTROL_ENABLE; m2p_set_control(edmac, control); + edmac->buffer = 0; + return 0; } @@ -331,21 +334,27 @@ static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac) return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3; } -static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac) +static void m2p_hw_synchronize(struct ep93xx_dma_chan *edmac) { + unsigned long flags; u32 control; + spin_lock_irqsave(&edmac->lock, flags); control = readl(edmac->regs + M2P_CONTROL); control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT); m2p_set_control(edmac, control); + spin_unlock_irqrestore(&edmac->lock, flags); while (m2p_channel_state(edmac) >= M2P_STATE_ON) - cpu_relax(); + schedule(); +} +static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac) +{ m2p_set_control(edmac, 0); - while (m2p_channel_state(edmac) == M2P_STATE_STALL) - cpu_relax(); + while (m2p_channel_state(edmac) != M2P_STATE_IDLE) + dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n"); } static void m2p_fill_desc(struct ep93xx_dma_chan *edmac) @@ -1161,6 +1170,26 @@ fail: } /** + * ep93xx_dma_synchronize - Synchronizes the termination of transfers to the + * current context. + * @chan: channel + * + * Synchronizes the DMA channel termination to the current context. When this + * function returns it is guaranteed that all transfers for previously issued + * descriptors have stopped and and it is safe to free the memory associated + * with them. Furthermore it is guaranteed that all complete callback functions + * for a previously submitted descriptor have finished running and it is safe to + * free resources accessed from within the complete callbacks. + */ +static void ep93xx_dma_synchronize(struct dma_chan *chan) +{ + struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); + + if (edmac->edma->hw_synchronize) + edmac->edma->hw_synchronize(edmac); +} + +/** * ep93xx_dma_terminate_all - terminate all transactions * @chan: channel * @@ -1323,6 +1352,7 @@ static int __init ep93xx_dma_probe(struct platform_device *pdev) dma_dev->device_prep_slave_sg = ep93xx_dma_prep_slave_sg; dma_dev->device_prep_dma_cyclic = ep93xx_dma_prep_dma_cyclic; dma_dev->device_config = ep93xx_dma_slave_config; + dma_dev->device_synchronize = ep93xx_dma_synchronize; dma_dev->device_terminate_all = ep93xx_dma_terminate_all; dma_dev->device_issue_pending = ep93xx_dma_issue_pending; dma_dev->device_tx_status = ep93xx_dma_tx_status; @@ -1340,6 +1370,7 @@ static int __init ep93xx_dma_probe(struct platform_device *pdev) } else { dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + edma->hw_synchronize = m2p_hw_synchronize; edma->hw_setup = m2p_hw_setup; edma->hw_shutdown = m2p_hw_shutdown; edma->hw_submit = m2p_hw_submit; diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 6ad4384b3fa8..ed8ed1192775 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -839,8 +839,6 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) goto free_resources; } - for (i = 0; i < IOAT_NUM_SRC_TEST; i++) - dma_srcs[i] = DMA_ERROR_CODE; for (i = 0; i < IOAT_NUM_SRC_TEST; i++) { dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); @@ -910,8 +908,6 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) xor_val_result = 1; - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = DMA_ERROR_CODE; for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); @@ -965,8 +961,6 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) op = IOAT_OP_XOR_VAL; xor_val_result = 0; - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = DMA_ERROR_CODE; for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); @@ -1017,18 +1011,14 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) goto free_resources; dma_unmap: if (op == IOAT_OP_XOR) { - if (dest_dma != DMA_ERROR_CODE) - dma_unmap_page(dev, dest_dma, PAGE_SIZE, - DMA_FROM_DEVICE); - for (i = 0; i < IOAT_NUM_SRC_TEST; i++) - if (dma_srcs[i] != DMA_ERROR_CODE) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, - DMA_TO_DEVICE); + while (--i >= 0) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, + DMA_TO_DEVICE); + dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); } else if (op == IOAT_OP_XOR_VAL) { - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - if (dma_srcs[i] != DMA_ERROR_CODE) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, - DMA_TO_DEVICE); + while (--i >= 0) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, + DMA_TO_DEVICE); } free_resources: dma->device_free_chan_resources(dma_chan); diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c index a28a01fcba67..f3e211f8f6c5 100644 --- a/drivers/dma/mv_xor_v2.c +++ b/drivers/dma/mv_xor_v2.c @@ -161,6 +161,7 @@ struct mv_xor_v2_device { struct mv_xor_v2_sw_desc *sw_desq; int desc_size; unsigned int npendings; + unsigned int hw_queue_idx; }; /** @@ -214,18 +215,6 @@ static void mv_xor_v2_set_data_buffers(struct mv_xor_v2_device *xor_dev, } /* - * Return the next available index in the DESQ. - */ -static int mv_xor_v2_get_desq_write_ptr(struct mv_xor_v2_device *xor_dev) -{ - /* read the index for the next available descriptor in the DESQ */ - u32 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ALLOC_OFF); - - return ((reg >> MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT) - & MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK); -} - -/* * notify the engine of new descriptors, and update the available index. */ static void mv_xor_v2_add_desc_to_desq(struct mv_xor_v2_device *xor_dev, @@ -257,22 +246,6 @@ static int mv_xor_v2_set_desc_size(struct mv_xor_v2_device *xor_dev) return MV_XOR_V2_EXT_DESC_SIZE; } -/* - * Set the IMSG threshold - */ -static inline -void mv_xor_v2_set_imsg_thrd(struct mv_xor_v2_device *xor_dev, int thrd_val) -{ - u32 reg; - - reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); - - reg &= (~MV_XOR_V2_DMA_IMSG_THRD_MASK << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); - reg |= (thrd_val << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); - - writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); -} - static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data) { struct mv_xor_v2_device *xor_dev = data; @@ -288,12 +261,6 @@ static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data) if (!ndescs) return IRQ_NONE; - /* - * Update IMSG threshold, to disable new IMSG interrupts until - * end of the tasklet - */ - mv_xor_v2_set_imsg_thrd(xor_dev, MV_XOR_V2_DESC_NUM); - /* schedule a tasklet to handle descriptors callbacks */ tasklet_schedule(&xor_dev->irq_tasklet); @@ -306,7 +273,6 @@ static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data) static dma_cookie_t mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx) { - int desq_ptr; void *dest_hw_desc; dma_cookie_t cookie; struct mv_xor_v2_sw_desc *sw_desc = @@ -322,15 +288,15 @@ mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx) spin_lock_bh(&xor_dev->lock); cookie = dma_cookie_assign(tx); - /* get the next available slot in the DESQ */ - desq_ptr = mv_xor_v2_get_desq_write_ptr(xor_dev); - /* copy the HW descriptor from the SW descriptor to the DESQ */ - dest_hw_desc = xor_dev->hw_desq_virt + desq_ptr; + dest_hw_desc = xor_dev->hw_desq_virt + xor_dev->hw_queue_idx; memcpy(dest_hw_desc, &sw_desc->hw_desc, xor_dev->desc_size); xor_dev->npendings++; + xor_dev->hw_queue_idx++; + if (xor_dev->hw_queue_idx >= MV_XOR_V2_DESC_NUM) + xor_dev->hw_queue_idx = 0; spin_unlock_bh(&xor_dev->lock); @@ -344,6 +310,7 @@ static struct mv_xor_v2_sw_desc * mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev) { struct mv_xor_v2_sw_desc *sw_desc; + bool found = false; /* Lock the channel */ spin_lock_bh(&xor_dev->lock); @@ -355,19 +322,23 @@ mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev) return NULL; } - /* get a free SW descriptor from the SW DESQ */ - sw_desc = list_first_entry(&xor_dev->free_sw_desc, - struct mv_xor_v2_sw_desc, free_list); + list_for_each_entry(sw_desc, &xor_dev->free_sw_desc, free_list) { + if (async_tx_test_ack(&sw_desc->async_tx)) { + found = true; + break; + } + } + + if (!found) { + spin_unlock_bh(&xor_dev->lock); + return NULL; + } + list_del(&sw_desc->free_list); /* Release the channel */ spin_unlock_bh(&xor_dev->lock); - /* set the async tx descriptor */ - dma_async_tx_descriptor_init(&sw_desc->async_tx, &xor_dev->dmachan); - sw_desc->async_tx.tx_submit = mv_xor_v2_tx_submit; - async_tx_ack(&sw_desc->async_tx); - return sw_desc; } @@ -389,6 +360,8 @@ mv_xor_v2_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, __func__, len, &src, &dest, flags); sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); + if (!sw_desc) + return NULL; sw_desc->async_tx.flags = flags; @@ -443,6 +416,8 @@ mv_xor_v2_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, __func__, src_cnt, len, &dest, flags); sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); + if (!sw_desc) + return NULL; sw_desc->async_tx.flags = flags; @@ -491,6 +466,8 @@ mv_xor_v2_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) container_of(chan, struct mv_xor_v2_device, dmachan); sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); + if (!sw_desc) + return NULL; /* set the HW descriptor */ hw_descriptor = &sw_desc->hw_desc; @@ -554,7 +531,6 @@ static void mv_xor_v2_tasklet(unsigned long data) { struct mv_xor_v2_device *xor_dev = (struct mv_xor_v2_device *) data; int pending_ptr, num_of_pending, i; - struct mv_xor_v2_descriptor *next_pending_hw_desc = NULL; struct mv_xor_v2_sw_desc *next_pending_sw_desc = NULL; dev_dbg(xor_dev->dmadev.dev, "%s %d\n", __func__, __LINE__); @@ -562,17 +538,10 @@ static void mv_xor_v2_tasklet(unsigned long data) /* get the pending descriptors parameters */ num_of_pending = mv_xor_v2_get_pending_params(xor_dev, &pending_ptr); - /* next HW descriptor */ - next_pending_hw_desc = xor_dev->hw_desq_virt + pending_ptr; - /* loop over free descriptors */ for (i = 0; i < num_of_pending; i++) { - - if (pending_ptr > MV_XOR_V2_DESC_NUM) - pending_ptr = 0; - - if (next_pending_sw_desc != NULL) - next_pending_hw_desc++; + struct mv_xor_v2_descriptor *next_pending_hw_desc = + xor_dev->hw_desq_virt + pending_ptr; /* get the SW descriptor related to the HW descriptor */ next_pending_sw_desc = @@ -608,15 +577,14 @@ static void mv_xor_v2_tasklet(unsigned long data) /* increment the next descriptor */ pending_ptr++; + if (pending_ptr >= MV_XOR_V2_DESC_NUM) + pending_ptr = 0; } if (num_of_pending != 0) { /* free the descriptores */ mv_xor_v2_free_desc_from_desq(xor_dev, num_of_pending); } - - /* Update IMSG threshold, to enable new IMSG interrupts */ - mv_xor_v2_set_imsg_thrd(xor_dev, 0); } /* @@ -648,9 +616,6 @@ static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev) writel((xor_dev->hw_desq & 0xFFFF00000000) >> 32, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF); - /* enable the DMA engine */ - writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); - /* * This is a temporary solution, until we activate the * SMMU. Set the attributes for reading & writing data buffers @@ -694,6 +659,9 @@ static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev) reg |= MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL; writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE); + /* enable the DMA engine */ + writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); + return 0; } @@ -725,6 +693,10 @@ static int mv_xor_v2_probe(struct platform_device *pdev) platform_set_drvdata(pdev, xor_dev); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + xor_dev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) return -EPROBE_DEFER; @@ -785,8 +757,15 @@ static int mv_xor_v2_probe(struct platform_device *pdev) /* add all SW descriptors to the free list */ for (i = 0; i < MV_XOR_V2_DESC_NUM; i++) { - xor_dev->sw_desq[i].idx = i; - list_add(&xor_dev->sw_desq[i].free_list, + struct mv_xor_v2_sw_desc *sw_desc = + xor_dev->sw_desq + i; + sw_desc->idx = i; + dma_async_tx_descriptor_init(&sw_desc->async_tx, + &xor_dev->dmachan); + sw_desc->async_tx.tx_submit = mv_xor_v2_tx_submit; + async_tx_ack(&sw_desc->async_tx); + + list_add(&sw_desc->free_list, &xor_dev->free_sw_desc); } diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index daf479cce691..8c1665c8fe33 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -916,12 +916,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( return NULL; } - /* When the port_window is used, one frame must cover the window */ - if (port_window) { - burst = port_window; - port_window_bytes = port_window * es_bytes[es]; - } - /* Now allocate and setup the descriptor. */ d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC); if (!d) @@ -931,6 +925,21 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->dev_addr = dev_addr; d->es = es; + /* When the port_window is used, one frame must cover the window */ + if (port_window) { + burst = port_window; + port_window_bytes = port_window * es_bytes[es]; + + d->ei = 1; + /* + * One frame covers the port_window and by configure + * the source frame index to be -1 * (port_window - 1) + * we instruct the sDMA that after a frame is processed + * it should move back to the start of the window. + */ + d->fi = -(port_window_bytes - 1); + } + d->ccr = c->ccr | CCR_SYNC_FRAME; if (dir == DMA_DEV_TO_MEM) { d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED; @@ -955,14 +964,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->ccr |= CCR_SRC_AMODE_POSTINC; if (port_window) { d->ccr |= CCR_DST_AMODE_DBLIDX; - d->ei = 1; - /* - * One frame covers the port_window and by configure - * the source frame index to be -1 * (port_window - 1) - * we instruct the sDMA that after a frame is processed - * it should move back to the start of the window. - */ - d->fi = -(port_window_bytes - 1); if (port_window_bytes >= 64) d->csdp |= CSDP_DST_BURST_64; @@ -1018,16 +1019,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( osg->addr = sg_dma_address(sgent); osg->en = en; osg->fn = sg_dma_len(sgent) / frame_bytes; - if (port_window && dir == DMA_DEV_TO_MEM) { - osg->ei = 1; - /* - * One frame covers the port_window and by configure - * the source frame index to be -1 * (port_window - 1) - * we instruct the sDMA that after a frame is processed - * it should move back to the start of the window. - */ - osg->fi = -(port_window_bytes - 1); - } if (d->using_ll) { osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC, diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 8b0da7fa520d..e90a7a0d760a 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -3008,7 +3008,8 @@ static int pl330_remove(struct amba_device *adev) for (i = 0; i < AMBA_NR_IRQS; i++) { irq = adev->irq[i]; - devm_free_irq(&adev->dev, irq, pl330); + if (irq) + devm_free_irq(&adev->dev, irq, pl330); } dma_async_device_unregister(&pl330->ddma); diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index db41795fe42a..bd261c9e9664 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -1287,6 +1287,9 @@ static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, if (desc->hwdescs.use) { dptr = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; + if (dptr == 0) + dptr = desc->nchunks; + dptr--; WARN_ON(dptr >= desc->nchunks); } else { running = desc->running; diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c index 72c649713ace..31a145154e9f 100644 --- a/drivers/dma/sh/usb-dmac.c +++ b/drivers/dma/sh/usb-dmac.c @@ -117,7 +117,7 @@ struct usb_dmac { #define USB_DMASWR 0x0008 #define USB_DMASWR_SWR (1 << 0) #define USB_DMAOR 0x0060 -#define USB_DMAOR_AE (1 << 2) +#define USB_DMAOR_AE (1 << 1) #define USB_DMAOR_DME (1 << 0) #define USB_DMASAR 0x0000 |
