diff options
Diffstat (limited to 'drivers/net/ethernet/stmicro/stmmac/stmmac_main.c')
| -rw-r--r-- | drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 1137 |
1 files changed, 838 insertions, 299 deletions
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index fd54c7c87485..5836b21edd7e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -36,6 +36,7 @@ #endif /* CONFIG_DEBUG_FS */ #include <linux/net_tstamp.h> #include <linux/phylink.h> +#include <linux/udp.h> #include <net/pkt_cls.h> #include "stmmac_ptp.h" #include "stmmac.h" @@ -45,7 +46,7 @@ #include "dwxgmac2.h" #include "hwif.h" -#define STMMAC_ALIGN(x) __ALIGN_KERNEL(x, SMP_CACHE_BYTES) +#define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) /* Module parameters */ @@ -105,7 +106,8 @@ MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); static irqreturn_t stmmac_interrupt(int irq, void *dev_id); #ifdef CONFIG_DEBUG_FS -static int stmmac_init_fs(struct net_device *dev); +static const struct net_device_ops stmmac_netdev_ops; +static void stmmac_init_fs(struct net_device *dev); static void stmmac_exit_fs(struct net_device *dev); #endif @@ -386,9 +388,8 @@ bool stmmac_eee_init(struct stmmac_priv *priv) /* Using PCS we cannot dial with the phy registers at this stage * so we do not support extra feature like EEE. */ - if ((priv->hw->pcs == STMMAC_PCS_RGMII) || - (priv->hw->pcs == STMMAC_PCS_TBI) || - (priv->hw->pcs == STMMAC_PCS_RTBI)) + if (priv->hw->pcs == STMMAC_PCS_TBI || + priv->hw->pcs == STMMAC_PCS_RTBI) return false; /* Check if MAC core supports the EEE feature. */ @@ -432,6 +433,7 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, struct sk_buff *skb) { struct skb_shared_hwtstamps shhwtstamp; + bool found = false; u64 ns = 0; if (!priv->hwts_tx_en) @@ -443,9 +445,13 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, /* check tx tstamp status */ if (stmmac_get_tx_timestamp_status(priv, p)) { - /* get the valid tstamp */ stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); + found = true; + } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { + found = true; + } + if (found) { memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); shhwtstamp.hwtstamp = ns_to_ktime(ns); @@ -453,8 +459,6 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, /* pass tstamp to stack */ skb_tstamp_tx(skb, &shhwtstamp); } - - return; } /* stmmac_get_rx_hwtstamp - get HW RX timestamps @@ -626,6 +630,7 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; ptp_v2 = PTP_TCR_TSVER2ENA; snap_type_sel = PTP_TCR_SNAPTYPSEL_1; + ts_event_en = PTP_TCR_TSEVNTENA; ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; ptp_over_ethernet = PTP_TCR_TSIPENA; @@ -828,15 +833,22 @@ static void stmmac_validate(struct phylink_config *config, phylink_set(mask, 1000baseT_Full); phylink_set(mask, 1000baseX_Full); } else if (priv->plat->has_xgmac) { - phylink_set(mac_supported, 2500baseT_Full); - phylink_set(mac_supported, 5000baseT_Full); - phylink_set(mac_supported, 10000baseSR_Full); - phylink_set(mac_supported, 10000baseLR_Full); - phylink_set(mac_supported, 10000baseER_Full); - phylink_set(mac_supported, 10000baseLRM_Full); - phylink_set(mac_supported, 10000baseT_Full); - phylink_set(mac_supported, 10000baseKX4_Full); - phylink_set(mac_supported, 10000baseKR_Full); + if (!max_speed || (max_speed >= 2500)) { + phylink_set(mac_supported, 2500baseT_Full); + phylink_set(mac_supported, 2500baseX_Full); + } + if (!max_speed || (max_speed >= 5000)) { + phylink_set(mac_supported, 5000baseT_Full); + } + if (!max_speed || (max_speed >= 10000)) { + phylink_set(mac_supported, 10000baseSR_Full); + phylink_set(mac_supported, 10000baseLR_Full); + phylink_set(mac_supported, 10000baseER_Full); + phylink_set(mac_supported, 10000baseLRM_Full); + phylink_set(mac_supported, 10000baseT_Full); + phylink_set(mac_supported, 10000baseKX4_Full); + phylink_set(mac_supported, 10000baseKR_Full); + } } /* Half-Duplex can only work with single queue */ @@ -856,10 +868,10 @@ static void stmmac_validate(struct phylink_config *config, __ETHTOOL_LINK_MODE_MASK_NBITS); } -static int stmmac_mac_link_state(struct phylink_config *config, - struct phylink_link_state *state) +static void stmmac_mac_pcs_get_state(struct phylink_config *config, + struct phylink_link_state *state) { - return -EOPNOTSUPP; + state->link = 0; } static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, @@ -953,7 +965,7 @@ static void stmmac_mac_link_up(struct phylink_config *config, static const struct phylink_mac_ops stmmac_phylink_mac_ops = { .validate = stmmac_validate, - .mac_link_state = stmmac_mac_link_state, + .mac_pcs_get_state = stmmac_mac_pcs_get_state, .mac_config = stmmac_mac_config, .mac_an_restart = stmmac_mac_an_restart, .mac_link_down = stmmac_mac_link_down, @@ -1026,7 +1038,7 @@ static int stmmac_init_phy(struct net_device *dev) static int stmmac_phy_setup(struct stmmac_priv *priv) { struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); - int mode = priv->plat->interface; + int mode = priv->plat->phy_interface; struct phylink *phylink; priv->phylink_config.dev = &priv->dev->dev; @@ -1077,6 +1089,8 @@ static void stmmac_display_tx_rings(struct stmmac_priv *priv) if (priv->extend_desc) head_tx = (void *)tx_q->dma_etx; + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + head_tx = (void *)tx_q->dma_entx; else head_tx = (void *)tx_q->dma_tx; @@ -1097,7 +1111,9 @@ static int stmmac_set_bfsize(int mtu, int bufsize) { int ret = bufsize; - if (mtu >= BUF_SIZE_4KiB) + if (mtu >= BUF_SIZE_8KiB) + ret = BUF_SIZE_16KiB; + else if (mtu >= BUF_SIZE_4KiB) ret = BUF_SIZE_8KiB; else if (mtu >= BUF_SIZE_2KiB) ret = BUF_SIZE_4KiB; @@ -1148,13 +1164,19 @@ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) int i; /* Clear the TX descriptors */ - for (i = 0; i < DMA_TX_SIZE; i++) + for (i = 0; i < DMA_TX_SIZE; i++) { + int last = (i == (DMA_TX_SIZE - 1)); + struct dma_desc *p; + if (priv->extend_desc) - stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic, - priv->mode, (i == DMA_TX_SIZE - 1)); + p = &tx_q->dma_etx[i].basic; + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + p = &tx_q->dma_entx[i].basic; else - stmmac_init_tx_desc(priv, &tx_q->dma_tx[i], - priv->mode, (i == DMA_TX_SIZE - 1)); + p = &tx_q->dma_tx[i]; + + stmmac_init_tx_desc(priv, p, priv->mode, last); + } } /** @@ -1198,6 +1220,17 @@ static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, if (!buf->page) return -ENOMEM; + if (priv->sph) { + buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool); + if (!buf->sec_page) + return -ENOMEM; + + buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); + stmmac_set_desc_sec_addr(priv, p, buf->sec_addr); + } else { + buf->sec_page = NULL; + } + buf->addr = page_pool_get_dma_addr(buf->page); stmmac_set_desc_addr(priv, p, buf->addr); if (priv->dma_buf_sz == BUF_SIZE_16KiB) @@ -1220,6 +1253,10 @@ static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) if (buf->page) page_pool_put_page(rx_q->page_pool, buf->page, false); buf->page = NULL; + + if (buf->sec_page) + page_pool_put_page(rx_q->page_pool, buf->sec_page, false); + buf->sec_page = NULL; } /** @@ -1266,19 +1303,9 @@ static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) struct stmmac_priv *priv = netdev_priv(dev); u32 rx_count = priv->plat->rx_queues_to_use; int ret = -ENOMEM; - int bfsize = 0; int queue; int i; - bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); - if (bfsize < 0) - bfsize = 0; - - if (bfsize < BUF_SIZE_16KiB) - bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); - - priv->dma_buf_sz = bfsize; - /* RX INITIALIZATION */ netif_dbg(priv, probe, priv->dev, "SKB addresses:\nskb\t\tskb data\tdma data\n"); @@ -1320,8 +1347,6 @@ static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) } } - buf_sz = bfsize; - return 0; err_init_rx_buffers: @@ -1365,7 +1390,7 @@ static int init_dma_tx_desc_rings(struct net_device *dev) if (priv->extend_desc) stmmac_mode_init(priv, tx_q->dma_etx, tx_q->dma_tx_phy, DMA_TX_SIZE, 1); - else + else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) stmmac_mode_init(priv, tx_q->dma_tx, tx_q->dma_tx_phy, DMA_TX_SIZE, 0); } @@ -1374,6 +1399,8 @@ static int init_dma_tx_desc_rings(struct net_device *dev) struct dma_desc *p; if (priv->extend_desc) p = &((tx_q->dma_etx + i)->basic); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + p = &((tx_q->dma_entx + i)->basic); else p = tx_q->dma_tx + i; @@ -1476,10 +1503,8 @@ static void free_dma_rx_desc_resources(struct stmmac_priv *priv) rx_q->dma_erx, rx_q->dma_rx_phy); kfree(rx_q->buf_pool); - if (rx_q->page_pool) { - page_pool_request_shutdown(rx_q->page_pool); + if (rx_q->page_pool) page_pool_destroy(rx_q->page_pool); - } } } @@ -1495,19 +1520,26 @@ static void free_dma_tx_desc_resources(struct stmmac_priv *priv) /* Free TX queue resources */ for (queue = 0; queue < tx_count; queue++) { struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + size_t size; + void *addr; /* Release the DMA TX socket buffers */ dma_free_tx_skbufs(priv, queue); - /* Free DMA regions of consistent memory previously allocated */ - if (!priv->extend_desc) - dma_free_coherent(priv->device, - DMA_TX_SIZE * sizeof(struct dma_desc), - tx_q->dma_tx, tx_q->dma_tx_phy); - else - dma_free_coherent(priv->device, DMA_TX_SIZE * - sizeof(struct dma_extended_desc), - tx_q->dma_etx, tx_q->dma_tx_phy); + if (priv->extend_desc) { + size = sizeof(struct dma_extended_desc); + addr = tx_q->dma_etx; + } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { + size = sizeof(struct dma_edesc); + addr = tx_q->dma_entx; + } else { + size = sizeof(struct dma_desc); + addr = tx_q->dma_tx; + } + + size *= DMA_TX_SIZE; + + dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); kfree(tx_q->tx_skbuff_dma); kfree(tx_q->tx_skbuff); @@ -1532,13 +1564,15 @@ static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) for (queue = 0; queue < rx_count; queue++) { struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; struct page_pool_params pp_params = { 0 }; + unsigned int num_pages; rx_q->queue_index = queue; rx_q->priv_data = priv; pp_params.flags = PP_FLAG_DMA_MAP; pp_params.pool_size = DMA_RX_SIZE; - pp_params.order = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE); + num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE); + pp_params.order = ilog2(num_pages); pp_params.nid = dev_to_node(priv->device); pp_params.dev = priv->device; pp_params.dma_dir = DMA_FROM_DEVICE; @@ -1598,6 +1632,8 @@ static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) /* TX queues buffers and DMA */ for (queue = 0; queue < tx_count; queue++) { struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + size_t size; + void *addr; tx_q->queue_index = queue; tx_q->priv_data = priv; @@ -1614,28 +1650,32 @@ static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) if (!tx_q->tx_skbuff) goto err_dma; - if (priv->extend_desc) { - tx_q->dma_etx = dma_alloc_coherent(priv->device, - DMA_TX_SIZE * sizeof(struct dma_extended_desc), - &tx_q->dma_tx_phy, - GFP_KERNEL); - if (!tx_q->dma_etx) - goto err_dma; - } else { - tx_q->dma_tx = dma_alloc_coherent(priv->device, - DMA_TX_SIZE * sizeof(struct dma_desc), - &tx_q->dma_tx_phy, - GFP_KERNEL); - if (!tx_q->dma_tx) - goto err_dma; - } + if (priv->extend_desc) + size = sizeof(struct dma_extended_desc); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + size = sizeof(struct dma_edesc); + else + size = sizeof(struct dma_desc); + + size *= DMA_TX_SIZE; + + addr = dma_alloc_coherent(priv->device, size, + &tx_q->dma_tx_phy, GFP_KERNEL); + if (!addr) + goto err_dma; + + if (priv->extend_desc) + tx_q->dma_etx = addr; + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + tx_q->dma_entx = addr; + else + tx_q->dma_tx = addr; } return 0; err_dma: free_dma_tx_desc_resources(priv); - return ret; } @@ -1867,6 +1907,8 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) if (priv->extend_desc) p = (struct dma_desc *)(tx_q->dma_etx + entry); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + p = &tx_q->dma_entx[entry].basic; else p = tx_q->dma_tx + entry; @@ -1948,7 +1990,7 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) /* We still have pending packets, let's call for a new scheduling */ if (tx_q->dirty_tx != tx_q->cur_tx) - mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(10)); + mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer)); __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); @@ -1965,23 +2007,18 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) { struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; - int i; netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); stmmac_stop_tx_dma(priv, chan); dma_free_tx_skbufs(priv, chan); - for (i = 0; i < DMA_TX_SIZE; i++) - if (priv->extend_desc) - stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic, - priv->mode, (i == DMA_TX_SIZE - 1)); - else - stmmac_init_tx_desc(priv, &tx_q->dma_tx[i], - priv->mode, (i == DMA_TX_SIZE - 1)); + stmmac_clear_tx_descriptors(priv, chan); tx_q->dirty_tx = 0; tx_q->cur_tx = 0; tx_q->mss = 0; netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); + stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, + tx_q->dma_tx_phy, chan); stmmac_start_tx_dma(priv, chan); priv->dev->stats.tx_errors++; @@ -2040,17 +2077,25 @@ static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan) int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, &priv->xstats, chan); struct stmmac_channel *ch = &priv->channel[chan]; + unsigned long flags; if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { if (napi_schedule_prep(&ch->rx_napi)) { - stmmac_disable_dma_irq(priv, priv->ioaddr, chan); + spin_lock_irqsave(&ch->lock, flags); + stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); + spin_unlock_irqrestore(&ch->lock, flags); __napi_schedule_irqoff(&ch->rx_napi); - status |= handle_tx; } } - if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) - napi_schedule_irqoff(&ch->tx_napi); + if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { + if (napi_schedule_prep(&ch->tx_napi)) { + spin_lock_irqsave(&ch->lock, flags); + stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); + spin_unlock_irqrestore(&ch->lock, flags); + __napi_schedule_irqoff(&ch->tx_napi); + } + } return status; } @@ -2245,14 +2290,14 @@ static void stmmac_tx_timer(struct timer_list *t) ch = &priv->channel[tx_q->queue_index]; - /* - * If NAPI is already running we can miss some events. Let's rearm - * the timer and try again. - */ - if (likely(napi_schedule_prep(&ch->tx_napi))) + if (likely(napi_schedule_prep(&ch->tx_napi))) { + unsigned long flags; + + spin_lock_irqsave(&ch->lock, flags); + stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); + spin_unlock_irqrestore(&ch->lock, flags); __napi_schedule(&ch->tx_napi); - else - mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(10)); + } } /** @@ -2417,6 +2462,22 @@ static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) } } +static void stmmac_mac_config_rss(struct stmmac_priv *priv) +{ + if (!priv->dma_cap.rssen || !priv->plat->rss_en) { + priv->rss.enable = false; + return; + } + + if (priv->dev->features & NETIF_F_RXHASH) + priv->rss.enable = true; + else + priv->rss.enable = false; + + stmmac_rss_configure(priv, priv->hw, &priv->rss, + priv->plat->rx_queues_to_use); +} + /** * stmmac_mtl_configuration - Configure MTL * @priv: driver private structure @@ -2461,6 +2522,10 @@ static void stmmac_mtl_configuration(struct stmmac_priv *priv) /* Set RX routing */ if (rx_queues_count > 1) stmmac_mac_config_rx_queues_routing(priv); + + /* Receive Side Scaling */ + if (rx_queues_count > 1) + stmmac_mac_config_rss(priv); } static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) @@ -2556,13 +2621,14 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; if (priv->use_riwt) { - ret = stmmac_rx_watchdog(priv, priv->ioaddr, MIN_DMA_RIWT, rx_cnt); - if (!ret) - priv->rx_riwt = MIN_DMA_RIWT; + if (!priv->rx_riwt) + priv->rx_riwt = DEF_DMA_RIWT; + + ret = stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt); } if (priv->hw->pcs) - stmmac_pcs_ctrl_ane(priv, priv->hw, 1, priv->hw->ps, 0); + stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); /* set TX and RX rings length */ stmmac_set_rings_length(priv); @@ -2573,6 +2639,24 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) stmmac_enable_tso(priv, priv->ioaddr, 1, chan); } + /* Enable Split Header */ + if (priv->sph && priv->hw->rx_csum) { + for (chan = 0; chan < rx_cnt; chan++) + stmmac_enable_sph(priv, priv->ioaddr, 1, chan); + } + + /* VLAN Tag Insertion */ + if (priv->dma_cap.vlins) + stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); + + /* TBS */ + for (chan = 0; chan < tx_cnt; chan++) { + struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; + int enable = tx_q->tbs & STMMAC_TBS_AVAIL; + + stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); + } + /* Start the ball rolling... */ stmmac_start_all_dma(priv); @@ -2598,11 +2682,11 @@ static void stmmac_hw_teardown(struct net_device *dev) static int stmmac_open(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); + int bfsize = 0; u32 chan; int ret; - if (priv->hw->pcs != STMMAC_PCS_RGMII && - priv->hw->pcs != STMMAC_PCS_TBI && + if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) { ret = stmmac_init_phy(dev); if (ret) { @@ -2617,9 +2701,28 @@ static int stmmac_open(struct net_device *dev) memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); priv->xstats.threshold = tc; - priv->dma_buf_sz = STMMAC_ALIGN(buf_sz); + bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); + if (bfsize < 0) + bfsize = 0; + + if (bfsize < BUF_SIZE_16KiB) + bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); + + priv->dma_buf_sz = bfsize; + buf_sz = bfsize; + priv->rx_copybreak = STMMAC_RX_COPYBREAK; + /* Earlier check for TBS */ + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { + struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; + int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; + + tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; + if (stmmac_enable_tbs(priv, priv->ioaddr, tbs_en, chan)) + tx_q->tbs &= ~STMMAC_TBS_AVAIL; + } + ret = alloc_dma_desc_resources(priv); if (ret < 0) { netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", @@ -2750,6 +2853,37 @@ static int stmmac_release(struct net_device *dev) return 0; } +static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, + struct stmmac_tx_queue *tx_q) +{ + u16 tag = 0x0, inner_tag = 0x0; + u32 inner_type = 0x0; + struct dma_desc *p; + + if (!priv->dma_cap.vlins) + return false; + if (!skb_vlan_tag_present(skb)) + return false; + if (skb->vlan_proto == htons(ETH_P_8021AD)) { + inner_tag = skb_vlan_tag_get(skb); + inner_type = STMMAC_VLAN_INSERT; + } + + tag = skb_vlan_tag_get(skb); + + if (tx_q->tbs & STMMAC_TBS_AVAIL) + p = &tx_q->dma_entx[tx_q->cur_tx].basic; + else + p = &tx_q->dma_tx[tx_q->cur_tx]; + + if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) + return false; + + stmmac_set_tx_owner(priv, p); + tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); + return true; +} + /** * stmmac_tso_allocator - close entry point of the driver * @priv: driver private structure @@ -2776,7 +2910,11 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); - desc = tx_q->dma_tx + tx_q->cur_tx; + + if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc = &tx_q->dma_entx[tx_q->cur_tx].basic; + else + desc = &tx_q->dma_tx[tx_q->cur_tx]; curr_addr = des + (total_len - tmp_len); if (priv->dma_cap.addr64 <= 32) @@ -2827,20 +2965,28 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) { struct dma_desc *desc, *first, *mss_desc = NULL; struct stmmac_priv *priv = netdev_priv(dev); + int desc_size, tmp_pay_len = 0, first_tx; int nfrags = skb_shinfo(skb)->nr_frags; u32 queue = skb_get_queue_mapping(skb); - unsigned int first_entry; + unsigned int first_entry, tx_packets; struct stmmac_tx_queue *tx_q; - int tmp_pay_len = 0; + bool has_vlan, set_ic; + u8 proto_hdr_len, hdr; u32 pay_len, mss; - u8 proto_hdr_len; dma_addr_t des; int i; tx_q = &priv->tx_queue[queue]; + first_tx = tx_q->cur_tx; /* Compute header lengths */ - proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { + proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); + hdr = sizeof(struct udphdr); + } else { + proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr = tcp_hdrlen(skb); + } /* Desc availability based on threshold should be enough safe */ if (unlikely(stmmac_tx_avail(priv, queue) < @@ -2862,7 +3008,11 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) /* set new MSS value if needed */ if (mss != tx_q->mss) { - mss_desc = tx_q->dma_tx + tx_q->cur_tx; + if (tx_q->tbs & STMMAC_TBS_AVAIL) + mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; + else + mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; + stmmac_set_mss(priv, mss_desc, mss); tx_q->mss = mss; tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); @@ -2870,18 +3020,27 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) } if (netif_msg_tx_queued(priv)) { - pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n", - __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss); + pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", + __func__, hdr, proto_hdr_len, pay_len, mss); pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, skb->data_len); } + /* Check if VLAN can be inserted by HW */ + has_vlan = stmmac_vlan_insert(priv, skb, tx_q); + first_entry = tx_q->cur_tx; WARN_ON(tx_q->tx_skbuff[first_entry]); - desc = tx_q->dma_tx + first_entry; + if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc = &tx_q->dma_entx[first_entry].basic; + else + desc = &tx_q->dma_tx[first_entry]; first = desc; + if (has_vlan) + stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); + /* first descriptor: fill Headers on Buf1 */ des = dma_map_single(priv->device, skb->data, skb_headlen(skb), DMA_TO_DEVICE); @@ -2903,6 +3062,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) } else { stmmac_set_desc_addr(priv, first, des); tmp_pay_len = pay_len; + des += proto_hdr_len; + pay_len = 0; } stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); @@ -2930,6 +3091,32 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) /* Only the last descriptor gets to point to the skb. */ tx_q->tx_skbuff[tx_q->cur_tx] = skb; + /* Manage tx mitigation */ + tx_packets = (tx_q->cur_tx + 1) - first_tx; + tx_q->tx_count_frames += tx_packets; + + if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) + set_ic = true; + else if (!priv->tx_coal_frames) + set_ic = false; + else if (tx_packets > priv->tx_coal_frames) + set_ic = true; + else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets) + set_ic = true; + else + set_ic = false; + + if (set_ic) { + if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc = &tx_q->dma_entx[tx_q->cur_tx].basic; + else + desc = &tx_q->dma_tx[tx_q->cur_tx]; + + tx_q->tx_count_frames = 0; + stmmac_set_tx_ic(priv, desc); + priv->xstats.tx_set_ic_bit++; + } + /* We've used all descriptors we need for this skb, however, * advance cur_tx so that it references a fresh descriptor. * ndo_start_xmit will fill this descriptor the next time it's @@ -2947,18 +3134,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) priv->xstats.tx_tso_frames++; priv->xstats.tx_tso_nfrags += nfrags; - /* Manage tx mitigation */ - tx_q->tx_count_frames += nfrags + 1; - if (likely(priv->tx_coal_frames > tx_q->tx_count_frames) && - !(priv->synopsys_id >= DWMAC_CORE_4_00 && - (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - priv->hwts_tx_en)) { - stmmac_tx_timer_arm(priv, queue); - } else { - tx_q->tx_count_frames = 0; - stmmac_set_tx_ic(priv, desc); - priv->xstats.tx_set_ic_bit++; - } + if (priv->sarc_type) + stmmac_set_desc_sarc(priv, first, priv->sarc_type); skb_tx_timestamp(skb); @@ -2974,7 +3151,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) proto_hdr_len, pay_len, 1, tx_q->tx_skbuff_dma[first_entry].last_segment, - tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); + hdr / 4, (skb->len - proto_hdr_len)); /* If context desc is used to change MSS */ if (mss_desc) { @@ -2997,17 +3174,20 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, tx_q->cur_tx, first, nfrags); - - stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0); - pr_info(">>> frame to be transmitted: "); print_pkt(skb->data, skb_headlen(skb)); } netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); - tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc)); + if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc_size = sizeof(struct dma_edesc); + else + desc_size = sizeof(struct dma_desc); + + tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); + stmmac_tx_timer_arm(priv, queue); return NETDEV_TX_OK; @@ -3028,26 +3208,31 @@ dma_map_err: */ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) { + unsigned int first_entry, tx_packets, enh_desc; struct stmmac_priv *priv = netdev_priv(dev); unsigned int nopaged_len = skb_headlen(skb); int i, csum_insertion = 0, is_jumbo = 0; u32 queue = skb_get_queue_mapping(skb); int nfrags = skb_shinfo(skb)->nr_frags; + int gso = skb_shinfo(skb)->gso_type; + struct dma_edesc *tbs_desc = NULL; + int entry, desc_size, first_tx; struct dma_desc *desc, *first; struct stmmac_tx_queue *tx_q; - unsigned int first_entry; - unsigned int enh_desc; + bool has_vlan, set_ic; dma_addr_t des; - int entry; tx_q = &priv->tx_queue[queue]; + first_tx = tx_q->cur_tx; if (priv->tx_path_in_lpi_mode) stmmac_disable_eee_mode(priv); /* Manage oversized TCP frames for GMAC4 device */ if (skb_is_gso(skb) && priv->tso) { - if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) + if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) + return stmmac_tso_xmit(skb, dev); + if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) return stmmac_tso_xmit(skb, dev); } @@ -3063,6 +3248,9 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_BUSY; } + /* Check if VLAN can be inserted by HW */ + has_vlan = stmmac_vlan_insert(priv, skb, tx_q); + entry = tx_q->cur_tx; first_entry = entry; WARN_ON(tx_q->tx_skbuff[first_entry]); @@ -3071,11 +3259,16 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) if (likely(priv->extend_desc)) desc = (struct dma_desc *)(tx_q->dma_etx + entry); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc = &tx_q->dma_entx[entry].basic; else desc = tx_q->dma_tx + entry; first = desc; + if (has_vlan) + stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); + enh_desc = priv->plat->enh_desc; /* To program the descriptors according to the size of the frame */ if (enh_desc) @@ -3097,6 +3290,8 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) if (likely(priv->extend_desc)) desc = (struct dma_desc *)(tx_q->dma_etx + entry); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc = &tx_q->dma_entx[entry].basic; else desc = tx_q->dma_tx + entry; @@ -3121,6 +3316,38 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) /* Only the last descriptor gets to point to the skb. */ tx_q->tx_skbuff[entry] = skb; + /* According to the coalesce parameter the IC bit for the latest + * segment is reset and the timer re-started to clean the tx status. + * This approach takes care about the fragments: desc is the first + * element in case of no SG. + */ + tx_packets = (entry + 1) - first_tx; + tx_q->tx_count_frames += tx_packets; + + if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) + set_ic = true; + else if (!priv->tx_coal_frames) + set_ic = false; + else if (tx_packets > priv->tx_coal_frames) + set_ic = true; + else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets) + set_ic = true; + else + set_ic = false; + + if (set_ic) { + if (likely(priv->extend_desc)) + desc = &tx_q->dma_etx[entry].basic; + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc = &tx_q->dma_entx[entry].basic; + else + desc = &tx_q->dma_tx[entry]; + + tx_q->tx_count_frames = 0; + stmmac_set_tx_ic(priv, desc); + priv->xstats.tx_set_ic_bit++; + } + /* We've used all descriptors we need for this skb, however, * advance cur_tx so that it references a fresh descriptor. * ndo_start_xmit will fill this descriptor the next time it's @@ -3130,20 +3357,11 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) tx_q->cur_tx = entry; if (netif_msg_pktdata(priv)) { - void *tx_head; - netdev_dbg(priv->dev, "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, entry, first, nfrags); - if (priv->extend_desc) - tx_head = (void *)tx_q->dma_etx; - else - tx_head = (void *)tx_q->dma_tx; - - stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false); - netdev_dbg(priv->dev, ">>> frame to be transmitted: "); print_pkt(skb->data, skb->len); } @@ -3156,22 +3374,8 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) dev->stats.tx_bytes += skb->len; - /* According to the coalesce parameter the IC bit for the latest - * segment is reset and the timer re-started to clean the tx status. - * This approach takes care about the fragments: desc is the first - * element in case of no SG. - */ - tx_q->tx_count_frames += nfrags + 1; - if (likely(priv->tx_coal_frames > tx_q->tx_count_frames) && - !(priv->synopsys_id >= DWMAC_CORE_4_00 && - (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - priv->hwts_tx_en)) { - stmmac_tx_timer_arm(priv, queue); - } else { - tx_q->tx_count_frames = 0; - stmmac_set_tx_ic(priv, desc); - priv->xstats.tx_set_ic_bit++; - } + if (priv->sarc_type) + stmmac_set_desc_sarc(priv, first, priv->sarc_type); skb_tx_timestamp(skb); @@ -3203,12 +3407,19 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) /* Prepare the first descriptor setting the OWN bit too */ stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, - csum_insertion, priv->mode, 1, last_segment, + csum_insertion, priv->mode, 0, last_segment, skb->len); - } else { - stmmac_set_tx_owner(priv, first); } + if (tx_q->tbs & STMMAC_TBS_EN) { + struct timespec64 ts = ns_to_timespec64(skb->tstamp); + + tbs_desc = &tx_q->dma_entx[first_entry]; + stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); + } + + stmmac_set_tx_owner(priv, first); + /* The own bit must be the latest setting done when prepare the * descriptor and then barrier is needed to make sure that * all is coherent before granting the DMA engine. @@ -3219,8 +3430,16 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) stmmac_enable_dma_transmission(priv, priv->ioaddr); - tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc)); + if (likely(priv->extend_desc)) + desc_size = sizeof(struct dma_extended_desc); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc_size = sizeof(struct dma_edesc); + else + desc_size = sizeof(struct dma_desc); + + tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); + stmmac_tx_timer_arm(priv, queue); return NETDEV_TX_OK; @@ -3292,6 +3511,17 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) break; } + if (priv->sph && !buf->sec_page) { + buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool); + if (!buf->sec_page) + break; + + buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); + + dma_sync_single_for_device(priv->device, buf->sec_addr, + len, DMA_FROM_DEVICE); + } + buf->addr = page_pool_get_dma_addr(buf->page); /* Sync whole allocation to device. This will invalidate old @@ -3301,11 +3531,18 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) DMA_FROM_DEVICE); stmmac_set_desc_addr(priv, p, buf->addr); + stmmac_set_desc_sec_addr(priv, p, buf->sec_addr); stmmac_refill_desc3(priv, rx_q, p); rx_q->rx_count_frames++; - rx_q->rx_count_frames %= priv->rx_coal_frames; - use_rx_wd = priv->use_riwt && rx_q->rx_count_frames; + rx_q->rx_count_frames += priv->rx_coal_frames; + if (rx_q->rx_count_frames > priv->rx_coal_frames) + rx_q->rx_count_frames = 0; + + use_rx_wd = !priv->rx_coal_frames; + use_rx_wd |= rx_q->rx_count_frames > 0; + if (!priv->use_riwt) + use_rx_wd = false; dma_wmb(); stmmac_set_rx_owner(priv, p, use_rx_wd); @@ -3318,6 +3555,55 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); } +static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, + struct dma_desc *p, + int status, unsigned int len) +{ + int ret, coe = priv->hw->rx_csum; + unsigned int plen = 0, hlen = 0; + + /* Not first descriptor, buffer is always zero */ + if (priv->sph && len) + return 0; + + /* First descriptor, get split header length */ + ret = stmmac_get_rx_header_len(priv, p, &hlen); + if (priv->sph && hlen) { + priv->xstats.rx_split_hdr_pkt_n++; + return hlen; + } + + /* First descriptor, not last descriptor and not split header */ + if (status & rx_not_ls) + return priv->dma_buf_sz; + + plen = stmmac_get_rx_frame_len(priv, p, coe); + + /* First descriptor and last descriptor and not split header */ + return min_t(unsigned int, priv->dma_buf_sz, plen); +} + +static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, + struct dma_desc *p, + int status, unsigned int len) +{ + int coe = priv->hw->rx_csum; + unsigned int plen = 0; + + /* Not split header, buffer is not available */ + if (!priv->sph) + return 0; + + /* Not last descriptor */ + if (status & rx_not_ls) + return priv->dma_buf_sz; + + plen = stmmac_get_rx_frame_len(priv, p, coe); + + /* Last descriptor */ + return plen - len; +} + /** * stmmac_rx - manage the receive process * @priv: driver private structure @@ -3330,9 +3616,10 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) { struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; struct stmmac_channel *ch = &priv->channel[queue]; + unsigned int count = 0, error = 0, len = 0; + int status = 0, coe = priv->hw->rx_csum; unsigned int next_entry = rx_q->cur_rx; - int coe = priv->hw->rx_csum; - unsigned int count = 0; + struct sk_buff *skb = NULL; if (netif_msg_rx_status(priv)) { void *rx_head; @@ -3346,10 +3633,30 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true); } while (count < limit) { + unsigned int buf1_len = 0, buf2_len = 0; + enum pkt_hash_types hash_type; struct stmmac_rx_buffer *buf; struct dma_desc *np, *p; - int entry, status; + int entry; + u32 hash; + + if (!count && rx_q->state_saved) { + skb = rx_q->state.skb; + error = rx_q->state.error; + len = rx_q->state.len; + } else { + rx_q->state_saved = false; + skb = NULL; + error = 0; + len = 0; + } + if (count >= limit) + break; + +read_again: + buf1_len = 0; + buf2_len = 0; entry = next_entry; buf = &rx_q->buf_pool[entry]; @@ -3365,8 +3672,6 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) if (unlikely(status & dma_own)) break; - count++; - rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE); next_entry = rx_q->cur_rx; @@ -3382,85 +3687,125 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) &priv->xstats, rx_q->dma_erx + entry); if (unlikely(status == discard_frame)) { page_pool_recycle_direct(rx_q->page_pool, buf->page); - priv->dev->stats.rx_errors++; buf->page = NULL; - } else { - struct sk_buff *skb; - int frame_len; - unsigned int des; + error = 1; + if (!priv->hwts_rx_en) + priv->dev->stats.rx_errors++; + } - stmmac_get_desc_addr(priv, p, &des); - frame_len = stmmac_get_rx_frame_len(priv, p, coe); + if (unlikely(error && (status & rx_not_ls))) + goto read_again; + if (unlikely(error)) { + dev_kfree_skb(skb); + skb = NULL; + count++; + continue; + } - /* If frame length is greater than skb buffer size - * (preallocated during init) then the packet is - * ignored - */ - if (frame_len > priv->dma_buf_sz) { - if (net_ratelimit()) - netdev_err(priv->dev, - "len %d larger than size (%d)\n", - frame_len, priv->dma_buf_sz); - priv->dev->stats.rx_length_errors++; - continue; - } + /* Buffer is good. Go on. */ - /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 - * Type frames (LLC/LLC-SNAP) - * - * llc_snap is never checked in GMAC >= 4, so this ACS - * feature is always disabled and packets need to be - * stripped manually. - */ - if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00) || - unlikely(status != llc_snap)) - frame_len -= ETH_FCS_LEN; - - if (netif_msg_rx_status(priv)) { - netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n", - p, entry, des); - netdev_dbg(priv->dev, "frame size %d, COE: %d\n", - frame_len, status); - } + prefetch(page_address(buf->page)); + if (buf->sec_page) + prefetch(page_address(buf->sec_page)); + + buf1_len = stmmac_rx_buf1_len(priv, p, status, len); + len += buf1_len; + buf2_len = stmmac_rx_buf2_len(priv, p, status, len); + len += buf2_len; + + /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 + * Type frames (LLC/LLC-SNAP) + * + * llc_snap is never checked in GMAC >= 4, so this ACS + * feature is always disabled and packets need to be + * stripped manually. + */ + if (likely(!(status & rx_not_ls)) && + (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || + unlikely(status != llc_snap))) { + if (buf2_len) + buf2_len -= ETH_FCS_LEN; + else + buf1_len -= ETH_FCS_LEN; + + len -= ETH_FCS_LEN; + } - skb = netdev_alloc_skb_ip_align(priv->dev, frame_len); - if (unlikely(!skb)) { + if (!skb) { + skb = napi_alloc_skb(&ch->rx_napi, buf1_len); + if (!skb) { priv->dev->stats.rx_dropped++; - continue; + count++; + goto drain_data; } dma_sync_single_for_cpu(priv->device, buf->addr, - frame_len, DMA_FROM_DEVICE); + buf1_len, DMA_FROM_DEVICE); skb_copy_to_linear_data(skb, page_address(buf->page), - frame_len); - skb_put(skb, frame_len); + buf1_len); + skb_put(skb, buf1_len); - if (netif_msg_pktdata(priv)) { - netdev_dbg(priv->dev, "frame received (%dbytes)", - frame_len); - print_pkt(skb->data, frame_len); - } + /* Data payload copied into SKB, page ready for recycle */ + page_pool_recycle_direct(rx_q->page_pool, buf->page); + buf->page = NULL; + } else if (buf1_len) { + dma_sync_single_for_cpu(priv->device, buf->addr, + buf1_len, DMA_FROM_DEVICE); + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + buf->page, 0, buf1_len, + priv->dma_buf_sz); + + /* Data payload appended into SKB */ + page_pool_release_page(rx_q->page_pool, buf->page); + buf->page = NULL; + } + + if (buf2_len) { + dma_sync_single_for_cpu(priv->device, buf->sec_addr, + buf2_len, DMA_FROM_DEVICE); + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + buf->sec_page, 0, buf2_len, + priv->dma_buf_sz); - stmmac_get_rx_hwtstamp(priv, p, np, skb); + /* Data payload appended into SKB */ + page_pool_release_page(rx_q->page_pool, buf->sec_page); + buf->sec_page = NULL; + } - stmmac_rx_vlan(priv->dev, skb); +drain_data: + if (likely(status & rx_not_ls)) + goto read_again; + if (!skb) + continue; - skb->protocol = eth_type_trans(skb, priv->dev); + /* Got entire packet into SKB. Finish it. */ - if (unlikely(!coe)) - skb_checksum_none_assert(skb); - else - skb->ip_summed = CHECKSUM_UNNECESSARY; + stmmac_get_rx_hwtstamp(priv, p, np, skb); + stmmac_rx_vlan(priv->dev, skb); + skb->protocol = eth_type_trans(skb, priv->dev); - napi_gro_receive(&ch->rx_napi, skb); + if (unlikely(!coe)) + skb_checksum_none_assert(skb); + else + skb->ip_summed = CHECKSUM_UNNECESSARY; - /* Data payload copied into SKB, page ready for recycle */ - page_pool_recycle_direct(rx_q->page_pool, buf->page); - buf->page = NULL; + if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) + skb_set_hash(skb, hash, hash_type); - priv->dev->stats.rx_packets++; - priv->dev->stats.rx_bytes += frame_len; - } + skb_record_rx_queue(skb, queue); + napi_gro_receive(&ch->rx_napi, skb); + skb = NULL; + + priv->dev->stats.rx_packets++; + priv->dev->stats.rx_bytes += len; + count++; + } + + if (status & rx_not_ls || skb) { + rx_q->state_saved = true; + rx_q->state.skb = skb; + rx_q->state.error = error; + rx_q->state.len = len; } stmmac_rx_refill(priv, queue); @@ -3481,8 +3826,14 @@ static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) priv->xstats.napi_poll++; work_done = stmmac_rx(priv, budget, chan); - if (work_done < budget && napi_complete_done(napi, work_done)) - stmmac_enable_dma_irq(priv, priv->ioaddr, chan); + if (work_done < budget && napi_complete_done(napi, work_done)) { + unsigned long flags; + + spin_lock_irqsave(&ch->lock, flags); + stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); + spin_unlock_irqrestore(&ch->lock, flags); + } + return work_done; } @@ -3491,7 +3842,6 @@ static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) struct stmmac_channel *ch = container_of(napi, struct stmmac_channel, tx_napi); struct stmmac_priv *priv = ch->priv_data; - struct stmmac_tx_queue *tx_q; u32 chan = ch->index; int work_done; @@ -3500,15 +3850,12 @@ static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) work_done = stmmac_tx_clean(priv, DMA_TX_SIZE, chan); work_done = min(work_done, budget); - if (work_done < budget) - napi_complete_done(napi, work_done); + if (work_done < budget && napi_complete_done(napi, work_done)) { + unsigned long flags; - /* Force transmission restart */ - tx_q = &priv->tx_queue[chan]; - if (tx_q->cur_tx != tx_q->dirty_tx) { - stmmac_enable_dma_transmission(priv, priv->ioaddr); - stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, - chan); + spin_lock_irqsave(&ch->lock, flags); + stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); + spin_unlock_irqrestore(&ch->lock, flags); } return work_done; @@ -3522,7 +3869,7 @@ static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) * netdev structure and arrange for the device to be reset to a sane state * in order to transmit a new packet. */ -static void stmmac_tx_timeout(struct net_device *dev) +static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct stmmac_priv *priv = netdev_priv(dev); @@ -3559,12 +3906,24 @@ static void stmmac_set_rx_mode(struct net_device *dev) static int stmmac_change_mtu(struct net_device *dev, int new_mtu) { struct stmmac_priv *priv = netdev_priv(dev); + int txfifosz = priv->plat->tx_fifo_size; + + if (txfifosz == 0) + txfifosz = priv->dma_cap.tx_fifo_size; + + txfifosz /= priv->plat->tx_queues_to_use; if (netif_running(dev)) { netdev_err(priv->dev, "must be stopped to change its MTU\n"); return -EBUSY; } + new_mtu = STMMAC_ALIGN(new_mtu); + + /* If condition true, FIFO is too small or MTU too large */ + if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) + return -EINVAL; + dev->mtu = new_mtu; netdev_update_features(dev); @@ -3606,6 +3965,8 @@ static int stmmac_set_features(struct net_device *netdev, netdev_features_t features) { struct stmmac_priv *priv = netdev_priv(netdev); + bool sph_en; + u32 chan; /* Keep the COE Type in case of csum is supporting */ if (features & NETIF_F_RXCSUM) @@ -3617,6 +3978,10 @@ static int stmmac_set_features(struct net_device *netdev, */ stmmac_rx_ipc(priv, priv->hw); + sph_en = (priv->hw->rx_csum > 0) && priv->sph; + for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) + stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); + return 0; } @@ -3755,12 +4120,17 @@ static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, struct stmmac_priv *priv = cb_priv; int ret = -EOPNOTSUPP; + if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) + return ret; + stmmac_disable_all_queues(priv); switch (type) { case TC_SETUP_CLSU32: - if (tc_cls_can_offload_and_chain0(priv->dev, type_data)) - ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); + ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); + break; + case TC_SETUP_CLSFLOWER: + ret = stmmac_tc_setup_cls(priv, priv, type_data); break; default: break; @@ -3785,6 +4155,10 @@ static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, priv, priv, true); case TC_SETUP_QDISC_CBS: return stmmac_tc_setup_cbs(priv, priv, type_data); + case TC_SETUP_QDISC_TAPRIO: + return stmmac_tc_setup_taprio(priv, priv, type_data); + case TC_SETUP_QDISC_ETF: + return stmmac_tc_setup_etf(priv, priv, type_data); default: return -EOPNOTSUPP; } @@ -3793,11 +4167,13 @@ static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, struct net_device *sb_dev) { - if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) { + int gso = skb_shinfo(skb)->gso_type; + + if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { /* - * There is no way to determine the number of TSO + * There is no way to determine the number of TSO/USO * capable Queues. Let's use always the Queue 0 - * because if TSO is supported then at least this + * because if TSO/USO is supported then at least this * one will be capable. */ return 0; @@ -3886,7 +4262,7 @@ static int stmmac_rings_status_show(struct seq_file *seq, void *v) seq_printf(seq, "Extended descriptor ring:\n"); sysfs_display_ring((void *)tx_q->dma_etx, DMA_TX_SIZE, 1, seq); - } else { + } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { seq_printf(seq, "Descriptor ring:\n"); sysfs_display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE, 0, seq); @@ -3955,61 +4331,182 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) priv->dma_cap.number_rx_channel); seq_printf(seq, "\tNumber of Additional TX channel: %d\n", priv->dma_cap.number_tx_channel); + seq_printf(seq, "\tNumber of Additional RX queues: %d\n", + priv->dma_cap.number_rx_queues); + seq_printf(seq, "\tNumber of Additional TX queues: %d\n", + priv->dma_cap.number_tx_queues); seq_printf(seq, "\tEnhanced descriptors: %s\n", (priv->dma_cap.enh_desc) ? "Y" : "N"); - + seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); + seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); + seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz); + seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); + seq_printf(seq, "\tNumber of PPS Outputs: %d\n", + priv->dma_cap.pps_out_num); + seq_printf(seq, "\tSafety Features: %s\n", + priv->dma_cap.asp ? "Y" : "N"); + seq_printf(seq, "\tFlexible RX Parser: %s\n", + priv->dma_cap.frpsel ? "Y" : "N"); + seq_printf(seq, "\tEnhanced Addressing: %d\n", + priv->dma_cap.addr64); + seq_printf(seq, "\tReceive Side Scaling: %s\n", + priv->dma_cap.rssen ? "Y" : "N"); + seq_printf(seq, "\tVLAN Hash Filtering: %s\n", + priv->dma_cap.vlhash ? "Y" : "N"); + seq_printf(seq, "\tSplit Header: %s\n", + priv->dma_cap.sphen ? "Y" : "N"); + seq_printf(seq, "\tVLAN TX Insertion: %s\n", + priv->dma_cap.vlins ? "Y" : "N"); + seq_printf(seq, "\tDouble VLAN: %s\n", + priv->dma_cap.dvlan ? "Y" : "N"); + seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", + priv->dma_cap.l3l4fnum); + seq_printf(seq, "\tARP Offloading: %s\n", + priv->dma_cap.arpoffsel ? "Y" : "N"); + seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", + priv->dma_cap.estsel ? "Y" : "N"); + seq_printf(seq, "\tFrame Preemption (FPE): %s\n", + priv->dma_cap.fpesel ? "Y" : "N"); + seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", + priv->dma_cap.tbssel ? "Y" : "N"); return 0; } DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); -static int stmmac_init_fs(struct net_device *dev) +/* Use network device events to rename debugfs file entries. + */ +static int stmmac_device_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct stmmac_priv *priv = netdev_priv(dev); + + if (dev->netdev_ops != &stmmac_netdev_ops) + goto done; + + switch (event) { + case NETDEV_CHANGENAME: + if (priv->dbgfs_dir) + priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, + priv->dbgfs_dir, + stmmac_fs_dir, + dev->name); + break; + } +done: + return NOTIFY_DONE; +} + +static struct notifier_block stmmac_notifier = { + .notifier_call = stmmac_device_event, +}; + +static void stmmac_init_fs(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); /* Create per netdev entries */ priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); - if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) { - netdev_err(priv->dev, "ERROR failed to create debugfs directory\n"); + /* Entry to report DMA RX/TX rings */ + debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, + &stmmac_rings_status_fops); - return -ENOMEM; + /* Entry to report the DMA HW features */ + debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, + &stmmac_dma_cap_fops); + + register_netdevice_notifier(&stmmac_notifier); +} + +static void stmmac_exit_fs(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + + unregister_netdevice_notifier(&stmmac_notifier); + debugfs_remove_recursive(priv->dbgfs_dir); +} +#endif /* CONFIG_DEBUG_FS */ + +static u32 stmmac_vid_crc32_le(__le16 vid_le) +{ + unsigned char *data = (unsigned char *)&vid_le; + unsigned char data_byte = 0; + u32 crc = ~0x0; + u32 temp = 0; + int i, bits; + + bits = get_bitmask_order(VLAN_VID_MASK); + for (i = 0; i < bits; i++) { + if ((i % 8) == 0) + data_byte = data[i / 8]; + + temp = ((crc & 1) ^ data_byte) & 1; + crc >>= 1; + data_byte >>= 1; + + if (temp) + crc ^= 0xedb88320; } - /* Entry to report DMA RX/TX rings */ - priv->dbgfs_rings_status = - debugfs_create_file("descriptors_status", 0444, - priv->dbgfs_dir, dev, - &stmmac_rings_status_fops); + return crc; +} - if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) { - netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n"); - debugfs_remove_recursive(priv->dbgfs_dir); +static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) +{ + u32 crc, hash = 0; + __le16 pmatch = 0; + int count = 0; + u16 vid = 0; + + for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { + __le16 vid_le = cpu_to_le16(vid); + crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; + hash |= (1 << crc); + count++; + } - return -ENOMEM; + if (!priv->dma_cap.vlhash) { + if (count > 2) /* VID = 0 always passes filter */ + return -EOPNOTSUPP; + + pmatch = cpu_to_le16(vid); + hash = 0; } - /* Entry to report the DMA HW features */ - priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", 0444, - priv->dbgfs_dir, - dev, &stmmac_dma_cap_fops); + return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); +} - if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) { - netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n"); - debugfs_remove_recursive(priv->dbgfs_dir); +static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) +{ + struct stmmac_priv *priv = netdev_priv(ndev); + bool is_double = false; + int ret; - return -ENOMEM; + if (be16_to_cpu(proto) == ETH_P_8021AD) + is_double = true; + + set_bit(vid, priv->active_vlans); + ret = stmmac_vlan_update(priv, is_double); + if (ret) { + clear_bit(vid, priv->active_vlans); + return ret; } - return 0; + return ret; } -static void stmmac_exit_fs(struct net_device *dev) +static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) { - struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + bool is_double = false; - debugfs_remove_recursive(priv->dbgfs_dir); + if (be16_to_cpu(proto) == ETH_P_8021AD) + is_double = true; + + clear_bit(vid, priv->active_vlans); + return stmmac_vlan_update(priv, is_double); } -#endif /* CONFIG_DEBUG_FS */ static const struct net_device_ops stmmac_netdev_ops = { .ndo_open = stmmac_open, @@ -4027,6 +4524,8 @@ static const struct net_device_ops stmmac_netdev_ops = { .ndo_poll_controller = stmmac_poll_controller, #endif .ndo_set_mac_address = stmmac_set_mac_address, + .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, }; static void stmmac_reset_subtask(struct stmmac_priv *priv) @@ -4175,8 +4674,8 @@ int stmmac_dvr_probe(struct device *device, { struct net_device *ndev = NULL; struct stmmac_priv *priv; - u32 queue, maxq; - int ret = 0; + u32 queue, rxq, maxq; + int i, ret = 0; ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); @@ -4255,16 +4754,31 @@ int stmmac_dvr_probe(struct device *device, if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; + if (priv->plat->has_gmac4) + ndev->hw_features |= NETIF_F_GSO_UDP_L4; priv->tso = true; dev_info(priv->device, "TSO feature enabled\n"); } + if (priv->dma_cap.sphen) { + ndev->hw_features |= NETIF_F_GRO; + priv->sph = true; + dev_info(priv->device, "SPH feature enabled\n"); + } + if (priv->dma_cap.addr64) { ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(priv->dma_cap.addr64)); if (!ret) { dev_info(priv->device, "Using %d bits DMA width\n", priv->dma_cap.addr64); + + /* + * If more than 32 bits can be addressed, make sure to + * enable enhanced addressing mode. + */ + if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) + priv->plat->dma_cfg->eame = true; } else { ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); if (ret) { @@ -4281,15 +4795,33 @@ int stmmac_dvr_probe(struct device *device, #ifdef STMMAC_VLAN_TAG_USED /* Both mac100 and gmac support receive VLAN tag detection */ ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; + if (priv->dma_cap.vlhash) { + ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; + ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; + } + if (priv->dma_cap.vlins) { + ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; + if (priv->dma_cap.dvlan) + ndev->features |= NETIF_F_HW_VLAN_STAG_TX; + } #endif priv->msg_enable = netif_msg_init(debug, default_msg_level); + /* Initialize RSS */ + rxq = priv->plat->rx_queues_to_use; + netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); + for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) + priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); + + if (priv->dma_cap.rssen && priv->plat->rss_en) + ndev->features |= NETIF_F_RXHASH; + /* MTU range: 46 - hw-specific max */ ndev->min_mtu = ETH_ZLEN - ETH_HLEN; - if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) - ndev->max_mtu = JUMBO_LEN; - else if (priv->plat->has_xgmac) + if (priv->plat->has_xgmac) ndev->max_mtu = XGMAC_JUMBO_LEN; + else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) + ndev->max_mtu = JUMBO_LEN; else ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu @@ -4312,6 +4844,7 @@ int stmmac_dvr_probe(struct device *device, for (queue = 0; queue < maxq; queue++) { struct stmmac_channel *ch = &priv->channel[queue]; + spin_lock_init(&ch->lock); ch->priv_data = priv; ch->index = queue; @@ -4341,8 +4874,7 @@ int stmmac_dvr_probe(struct device *device, stmmac_check_pcs_mode(priv); - if (priv->hw->pcs != STMMAC_PCS_RGMII && - priv->hw->pcs != STMMAC_PCS_TBI && + if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) { /* MDIO bus Registration */ ret = stmmac_mdio_register(ndev); @@ -4368,10 +4900,7 @@ int stmmac_dvr_probe(struct device *device, } #ifdef CONFIG_DEBUG_FS - ret = stmmac_init_fs(ndev); - if (ret < 0) - netdev_warn(priv->dev, "%s: failed debugFS registration\n", - __func__); + stmmac_init_fs(ndev); #endif return ret; @@ -4379,8 +4908,7 @@ int stmmac_dvr_probe(struct device *device, error_netdev_register: phylink_destroy(priv->phylink); error_phy_setup: - if (priv->hw->pcs != STMMAC_PCS_RGMII && - priv->hw->pcs != STMMAC_PCS_TBI && + if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) stmmac_mdio_unregister(ndev); error_mdio_register: @@ -4425,8 +4953,7 @@ int stmmac_dvr_remove(struct device *dev) reset_control_assert(priv->plat->stmmac_rst); clk_disable_unprepare(priv->plat->pclk); clk_disable_unprepare(priv->plat->stmmac_clk); - if (priv->hw->pcs != STMMAC_PCS_RGMII && - priv->hw->pcs != STMMAC_PCS_TBI && + if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) stmmac_mdio_unregister(ndev); destroy_workqueue(priv->wq); @@ -4447,11 +4974,12 @@ int stmmac_suspend(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct stmmac_priv *priv = netdev_priv(ndev); + u32 chan; if (!ndev || !netif_running(ndev)) return 0; - phylink_stop(priv->phylink); + phylink_mac_change(priv->phylink, false); mutex_lock(&priv->lock); @@ -4460,6 +4988,9 @@ int stmmac_suspend(struct device *dev) stmmac_disable_all_queues(priv); + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) + del_timer_sync(&priv->tx_queue[chan].txtimer); + /* Stop TX/RX DMA */ stmmac_stop_all_dma(priv); @@ -4468,11 +4999,19 @@ int stmmac_suspend(struct device *dev) stmmac_pmt(priv, priv->hw, priv->wolopts); priv->irq_wake = 1; } else { + mutex_unlock(&priv->lock); + rtnl_lock(); + phylink_stop(priv->phylink); + rtnl_unlock(); + mutex_lock(&priv->lock); + stmmac_mac_set(priv, priv->ioaddr, false); pinctrl_pm_select_sleep_state(priv->device); /* Disable clock in case of PWM is off */ - clk_disable(priv->plat->pclk); - clk_disable(priv->plat->stmmac_clk); + if (priv->plat->clk_ptp_ref) + clk_disable_unprepare(priv->plat->clk_ptp_ref); + clk_disable_unprepare(priv->plat->pclk); + clk_disable_unprepare(priv->plat->stmmac_clk); } mutex_unlock(&priv->lock); @@ -4535,8 +5074,10 @@ int stmmac_resume(struct device *dev) } else { pinctrl_pm_select_default_state(priv->device); /* enable the clk previously disabled */ - clk_enable(priv->plat->stmmac_clk); - clk_enable(priv->plat->pclk); + clk_prepare_enable(priv->plat->stmmac_clk); + clk_prepare_enable(priv->plat->pclk); + if (priv->plat->clk_ptp_ref) + clk_prepare_enable(priv->plat->clk_ptp_ref); /* reset the phy so that it's ready */ if (priv->mii) stmmac_mdio_reset(priv->mii); @@ -4560,7 +5101,13 @@ int stmmac_resume(struct device *dev) mutex_unlock(&priv->lock); - phylink_start(priv->phylink); + if (!device_may_wakeup(priv->device)) { + rtnl_lock(); + phylink_start(priv->phylink); + rtnl_unlock(); + } + + phylink_mac_change(priv->phylink, true); return 0; } @@ -4617,16 +5164,8 @@ static int __init stmmac_init(void) { #ifdef CONFIG_DEBUG_FS /* Create debugfs main directory if it doesn't exist yet */ - if (!stmmac_fs_dir) { + if (!stmmac_fs_dir) stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); - - if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) { - pr_err("ERROR %s, debugfs create directory failed\n", - STMMAC_RESOURCE_NAME); - - return -ENOMEM; - } - } #endif return 0; |
