summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_port.c14
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h5
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en.h6
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_main.c45
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rx.c5
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_tx.c92
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c4
7 files changed, 93 insertions, 78 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_port.c b/drivers/net/ethernet/mellanox/mlx4/en_port.c
index 73f6277d9ac1..ee99e67187f5 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_port.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_port.c
@@ -203,6 +203,20 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset)
priv->port_stats.tso_packets += ring->tso_packets;
priv->port_stats.xmit_more += ring->xmit_more;
}
+ if (mlx4_is_master(mdev->dev)) {
+ stats->rx_packets = en_stats_adder(&mlx4_en_stats->RTOT_prio_0,
+ &mlx4_en_stats->RTOT_prio_1,
+ NUM_PRIORITIES);
+ stats->tx_packets = en_stats_adder(&mlx4_en_stats->TTOT_prio_0,
+ &mlx4_en_stats->TTOT_prio_1,
+ NUM_PRIORITIES);
+ stats->rx_bytes = en_stats_adder(&mlx4_en_stats->ROCT_prio_0,
+ &mlx4_en_stats->ROCT_prio_1,
+ NUM_PRIORITIES);
+ stats->tx_bytes = en_stats_adder(&mlx4_en_stats->TOCT_prio_0,
+ &mlx4_en_stats->TOCT_prio_1,
+ NUM_PRIORITIES);
+ }
/* net device stats */
stats->rx_errors = be64_to_cpu(mlx4_en_stats->PCS) +
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h
index c5c1de9cf2ce..7fd466c0b929 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h
@@ -79,8 +79,7 @@ struct mlx4_en_flow_stats_tx {
#define NUM_FLOW_STATS (NUM_FLOW_STATS_RX + NUM_FLOW_STATS_TX + \
NUM_FLOW_PRIORITY_STATS_TX + \
- NUM_FLOW_PRIORITY_STATS_RX + \
- NUM_PF_STATS)
+ NUM_FLOW_PRIORITY_STATS_RX)
struct mlx4_en_stat_out_flow_control_mbox {
/* Total number of PAUSE frames received from the far-end port */
@@ -108,7 +107,7 @@ enum {
};
#define NUM_ALL_STATS (NUM_MAIN_STATS + NUM_PORT_STATS + NUM_PKT_STATS + \
- NUM_FLOW_STATS + NUM_PERF_STATS)
+ NUM_FLOW_STATS + NUM_PERF_STATS + NUM_PF_STATS)
#define MLX4_FIND_NETDEV_STAT(n) (offsetof(struct net_device_stats, n) / \
sizeof(((struct net_device_stats *)0)->n))
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index e14120eccf04..3d23bd657e3c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -208,7 +208,6 @@ enum cq_flags {
struct mlx5e_cq {
/* data path - accessed per cqe */
struct mlx5_cqwq wq;
- void *sqrq;
unsigned long flags;
/* data path - accessed per napi poll */
@@ -316,6 +315,7 @@ struct mlx5e_channel {
__be32 mkey_be;
u8 num_tc;
unsigned long flags;
+ int tc_to_txq_map[MLX5E_MAX_NUM_TC];
/* control */
struct mlx5e_priv *priv;
@@ -379,10 +379,9 @@ struct mlx5e_flow_table {
struct mlx5e_priv {
/* priv data path fields - start */
- int order_base_2_num_channels;
- int queue_mapping_channel_mask;
int num_tc;
int default_vlan_prio;
+ struct mlx5e_sq **txq_to_sq_map;
/* priv data path fields - end */
unsigned long state;
@@ -460,7 +459,6 @@ void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw);
u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
void *accel_priv, select_queue_fallback_t fallback);
netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
-netdev_tx_t mlx5e_xmit_multi_tc(struct sk_buff *skb, struct net_device *dev);
void mlx5e_completion_event(struct mlx5_core_cq *mcq);
void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 9a48d8eac0fc..40206da1f9d7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -345,7 +345,6 @@ static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
MLX5_SET(rqc, rqc, flush_in_error_en, 1);
- MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
PAGE_SHIFT);
MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
@@ -496,6 +495,7 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
void *sqc = param->sqc;
void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
+ int txq_ix;
int err;
err = mlx5_alloc_map_uar(mdev, &sq->uar);
@@ -515,14 +515,15 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
if (err)
goto err_sq_wq_destroy;
- sq->txq = netdev_get_tx_queue(priv->netdev,
- c->ix + tc * priv->params.num_channels);
+ txq_ix = c->ix + tc * priv->params.num_channels;
+ sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
sq->pdev = c->pdev;
sq->mkey_be = c->mkey_be;
sq->channel = c;
sq->tc = tc;
sq->edge = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
+ priv->txq_to_sq_map[txq_ix] = sq;
return 0;
@@ -852,8 +853,6 @@ static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
priv->params.tx_cq_moderation_pkts);
if (err)
goto err_close_tx_cqs;
-
- c->sq[tc].cq.sqrq = &c->sq[tc];
}
return 0;
@@ -902,6 +901,15 @@ static void mlx5e_close_sqs(struct mlx5e_channel *c)
mlx5e_close_sq(&c->sq[tc]);
}
+static void mlx5e_build_tc_to_txq_map(struct mlx5e_channel *c,
+ int num_channels)
+{
+ int i;
+
+ for (i = 0; i < MLX5E_MAX_NUM_TC; i++)
+ c->tc_to_txq_map[i] = c->ix + i * num_channels;
+}
+
static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
struct mlx5e_channel_param *cparam,
struct mlx5e_channel **cp)
@@ -923,6 +931,8 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
c->mkey_be = cpu_to_be32(priv->mr.key);
c->num_tc = priv->num_tc;
+ mlx5e_build_tc_to_txq_map(c, priv->params.num_channels);
+
netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
err = mlx5e_open_tx_cqs(c, cparam);
@@ -934,7 +944,6 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
priv->params.rx_cq_moderation_pkts);
if (err)
goto err_close_tx_cqs;
- c->rq.cq.sqrq = &c->rq;
napi_enable(&c->napi);
@@ -1050,14 +1059,18 @@ static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
static int mlx5e_open_channels(struct mlx5e_priv *priv)
{
struct mlx5e_channel_param cparam;
- int err;
+ int err = -ENOMEM;
int i;
int j;
priv->channel = kcalloc(priv->params.num_channels,
sizeof(struct mlx5e_channel *), GFP_KERNEL);
- if (!priv->channel)
- return -ENOMEM;
+
+ priv->txq_to_sq_map = kcalloc(priv->params.num_channels * priv->num_tc,
+ sizeof(struct mlx5e_sq *), GFP_KERNEL);
+
+ if (!priv->channel || !priv->txq_to_sq_map)
+ goto err_free_txq_to_sq_map;
mlx5e_build_channel_param(priv, &cparam);
for (i = 0; i < priv->params.num_channels; i++) {
@@ -1078,6 +1091,8 @@ err_close_channels:
for (i--; i >= 0; i--)
mlx5e_close_channel(priv->channel[i]);
+err_free_txq_to_sq_map:
+ kfree(priv->txq_to_sq_map);
kfree(priv->channel);
return err;
@@ -1090,6 +1105,7 @@ static void mlx5e_close_channels(struct mlx5e_priv *priv)
for (i = 0; i < priv->params.num_channels; i++)
mlx5e_close_channel(priv->channel[i]);
+ kfree(priv->txq_to_sq_map);
kfree(priv->channel);
}
@@ -1384,8 +1400,7 @@ int mlx5e_open_locked(struct net_device *netdev)
int num_txqs;
int err;
- num_txqs = roundup_pow_of_two(priv->params.num_channels) *
- priv->params.num_tc;
+ num_txqs = priv->params.num_channels * priv->params.num_tc;
netif_set_real_num_tx_queues(netdev, num_txqs);
netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
@@ -1693,9 +1708,6 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
priv->mdev = mdev;
priv->netdev = netdev;
priv->params.num_channels = num_comp_vectors;
- priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
- priv->queue_mapping_channel_mask =
- roundup_pow_of_two(num_comp_vectors) - 1;
priv->num_tc = priv->params.num_tc;
priv->default_vlan_prio = priv->params.default_vlan_prio;
@@ -1723,7 +1735,6 @@ static void mlx5e_build_netdev(struct net_device *netdev)
if (priv->num_tc > 1) {
mlx5e_netdev_ops.ndo_select_queue = mlx5e_select_queue;
- mlx5e_netdev_ops.ndo_start_xmit = mlx5e_xmit_multi_tc;
}
netdev->netdev_ops = &mlx5e_netdev_ops;
@@ -1793,9 +1804,7 @@ static void *mlx5e_create_netdev(struct mlx5_core_dev *mdev)
if (mlx5e_check_required_hca_cap(mdev))
return NULL;
- netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
- roundup_pow_of_two(ncv) * MLX5E_MAX_NUM_TC,
- ncv);
+ netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), ncv, ncv);
if (!netdev) {
mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
return NULL;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 06e7c744ed4a..9a9374131f5b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -191,7 +191,7 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
bool mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
{
- struct mlx5e_rq *rq = cq->sqrq;
+ struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
int i;
/* avoid accessing cq (dma coherent memory) if not needed */
@@ -209,10 +209,13 @@ bool mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
if (!cqe)
break;
+ mlx5_cqwq_pop(&cq->wq);
+
wqe_counter_be = cqe->wqe_counter;
wqe_counter = be16_to_cpu(wqe_counter_be);
wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
skb = rq->skb[wqe_counter];
+ prefetch(skb->data);
rq->skb[wqe_counter] = NULL;
dma_unmap_single(rq->pdev,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index bac268a670f4..03f28f438e55 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -106,7 +106,7 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
priv->default_vlan_prio;
int tc = netdev_get_prio_tc_map(dev, up);
- return (tc << priv->order_base_2_num_channels) | channel_ix;
+ return priv->channel[channel_ix]->tc_to_txq_map[tc];
}
static inline u16 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq,
@@ -143,20 +143,17 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
if (skb_is_gso(skb)) {
u32 payload_len;
- int num_pkts;
eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
opcode = MLX5_OPCODE_LSO;
ihs = skb_transport_offset(skb) + tcp_hdrlen(skb);
payload_len = skb->len - ihs;
- num_pkts = (payload_len / skb_shinfo(skb)->gso_size) +
- !!(payload_len % skb_shinfo(skb)->gso_size);
MLX5E_TX_SKB_CB(skb)->num_bytes = skb->len +
- (num_pkts - 1) * ihs;
+ (skb_shinfo(skb)->gso_segs - 1) * ihs;
sq->stats.tso_packets++;
sq->stats.tso_bytes += payload_len;
} else {
- ihs = mlx5e_get_inline_hdr_size(sq, skb);
+ ihs = mlx5e_get_inline_hdr_size(sq, skb);
MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len,
ETH_ZLEN);
}
@@ -164,7 +161,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
skb_copy_from_linear_data(skb, eseg->inline_hdr_start, ihs);
skb_pull_inline(skb, ihs);
- eseg->inline_hdr_sz = cpu_to_be16(ihs);
+ eseg->inline_hdr_sz = cpu_to_be16(ihs);
ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start),
@@ -211,9 +208,8 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
ds_cnt += MLX5E_TX_SKB_CB(skb)->num_dma;
- cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
- cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
- cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
+ cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
+ cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
sq->skb[pi] = skb;
@@ -228,8 +224,10 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
sq->stats.stopped++;
}
- if (!skb->xmit_more || netif_xmit_stopped(sq->txq))
+ if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) {
+ cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
mlx5e_tx_notify_hw(sq, wqe);
+ }
/* fill sq edge with nops to avoid wqe wrap around */
while ((sq->pc & wq->sz_m1) > sq->edge)
@@ -250,21 +248,7 @@ dma_unmap_wqe_err:
netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct mlx5e_priv *priv = netdev_priv(dev);
- int ix = skb->queue_mapping;
- int tc = 0;
- struct mlx5e_channel *c = priv->channel[ix];
- struct mlx5e_sq *sq = &c->sq[tc];
-
- return mlx5e_sq_xmit(sq, skb);
-}
-
-netdev_tx_t mlx5e_xmit_multi_tc(struct sk_buff *skb, struct net_device *dev)
-{
- struct mlx5e_priv *priv = netdev_priv(dev);
- int ix = skb->queue_mapping & priv->queue_mapping_channel_mask;
- int tc = skb->queue_mapping >> priv->order_base_2_num_channels;
- struct mlx5e_channel *c = priv->channel[ix];
- struct mlx5e_sq *sq = &c->sq[tc];
+ struct mlx5e_sq *sq = priv->txq_to_sq_map[skb_get_queue_mapping(skb)];
return mlx5e_sq_xmit(sq, skb);
}
@@ -282,7 +266,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq)
if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags))
return false;
- sq = cq->sqrq;
+ sq = container_of(cq, struct mlx5e_sq, cq);
npkts = 0;
nbytes = 0;
@@ -297,38 +281,48 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq)
for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
struct mlx5_cqe64 *cqe;
- struct sk_buff *skb;
- u16 ci;
- int j;
+ u16 wqe_counter;
+ bool last_wqe;
cqe = mlx5e_get_cqe(cq);
if (!cqe)
break;
- ci = sqcc & sq->wq.sz_m1;
- skb = sq->skb[ci];
+ mlx5_cqwq_pop(&cq->wq);
+
+ wqe_counter = be16_to_cpu(cqe->wqe_counter);
+
+ do {
+ struct sk_buff *skb;
+ u16 ci;
+ int j;
+
+ last_wqe = (sqcc == wqe_counter);
- if (unlikely(!skb)) { /* nop */
- sq->stats.nop++;
- sqcc++;
- goto free_skb;
- }
+ ci = sqcc & sq->wq.sz_m1;
+ skb = sq->skb[ci];
- for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) {
- dma_addr_t addr;
- u32 size;
+ if (unlikely(!skb)) { /* nop */
+ sq->stats.nop++;
+ sqcc++;
+ continue;
+ }
- mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size);
- dma_fifo_cc++;
- dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE);
- }
+ for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) {
+ dma_addr_t addr;
+ u32 size;
- npkts++;
- nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes;
- sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;
+ mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size);
+ dma_fifo_cc++;
+ dma_unmap_single(sq->pdev, addr, size,
+ DMA_TO_DEVICE);
+ }
-free_skb:
- dev_kfree_skb(skb);
+ npkts++;
+ nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes;
+ sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;
+ dev_kfree_skb(skb);
+ } while (!last_wqe);
}
mlx5_cqwq_update_db_record(&cq->wq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 088bc424157c..2c7cb6755d1d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -43,8 +43,6 @@ struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq)
if (cqe_ownership_bit != sw_ownership_val)
return NULL;
- mlx5_cqwq_pop(wq);
-
/* ensure cqe content is read after cqe ownership bit */
rmb();
@@ -65,7 +63,7 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
busy |= mlx5e_poll_rx_cq(&c->rq.cq, budget);
- busy |= mlx5e_post_rx_wqes(c->rq.cq.sqrq);
+ busy |= mlx5e_post_rx_wqes(&c->rq);
if (busy)
return budget;
OpenPOWER on IntegriCloud