diff options
Diffstat (limited to 'drivers/net/ethernet/hisilicon/hns3/hns3_enet.c')
| -rw-r--r-- | drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 1251 |
1 files changed, 762 insertions, 489 deletions
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 310afa708831..acb796cc10d0 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -24,10 +24,22 @@ #include "hnae3.h" #include "hns3_enet.h" +/* All hns3 tracepoints are defined by the include below, which + * must be included exactly once across the whole kernel with + * CREATE_TRACE_POINTS defined + */ +#define CREATE_TRACE_POINTS +#include "hns3_trace.h" #define hns3_set_field(origin, shift, val) ((origin) |= ((val) << (shift))) #define hns3_tx_bd_count(S) DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE) +#define hns3_rl_err(fmt, ...) \ + do { \ + if (net_ratelimit()) \ + netdev_err(fmt, ##__VA_ARGS__); \ + } while (0) + static void hns3_clear_all_ring(struct hnae3_handle *h, bool force); static void hns3_remove_hw_addr(struct net_device *netdev); @@ -45,6 +57,11 @@ MODULE_PARM_DESC(debug, " Network interface message level setting"); #define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \ NETIF_MSG_IFDOWN | NETIF_MSG_IFUP) +#define HNS3_INNER_VLAN_TAG 1 +#define HNS3_OUTER_VLAN_TAG 2 + +#define HNS3_MIN_TX_LEN 33U + /* hns3_pci_tbl - PCI Device ID Table * * Last entry must be all 0s @@ -118,18 +135,21 @@ static int hns3_nic_init_irq(struct hns3_nic_priv *priv) continue; if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) { - snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1, - "%s-%s-%d", priv->netdev->name, "TxRx", - txrx_int_idx++); + snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN, + "%s-%s-%s-%d", hns3_driver_name, + pci_name(priv->ae_handle->pdev), + "TxRx", txrx_int_idx++); txrx_int_idx++; } else if (tqp_vectors->rx_group.ring) { - snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1, - "%s-%s-%d", priv->netdev->name, "Rx", - rx_int_idx++); + snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN, + "%s-%s-%s-%d", hns3_driver_name, + pci_name(priv->ae_handle->pdev), + "Rx", rx_int_idx++); } else if (tqp_vectors->tx_group.ring) { - snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1, - "%s-%s-%d", priv->netdev->name, "Tx", - tx_int_idx++); + snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN, + "%s-%s-%s-%d", hns3_driver_name, + pci_name(priv->ae_handle->pdev), + "Tx", tx_int_idx++); } else { /* Skip this unused q_vector */ continue; @@ -146,6 +166,8 @@ static int hns3_nic_init_irq(struct hns3_nic_priv *priv) return ret; } + disable_irq(tqp_vectors->vector_irq); + irq_set_affinity_hint(tqp_vectors->vector_irq, &tqp_vectors->affinity_mask); @@ -164,6 +186,7 @@ static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector, static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector) { napi_enable(&tqp_vector->napi); + enable_irq(tqp_vector->vector_irq); /* enable vector */ hns3_mask_vector_irq(tqp_vector, 1); @@ -220,9 +243,9 @@ static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector, /* initialize the configuration for interrupt coalescing. * 1. GL (Interrupt Gap Limiter) * 2. RL (Interrupt Rate Limiter) + * + * Default: enable interrupt coalescing self-adaptive and GL */ - - /* Default: enable interrupt coalescing self-adaptive and GL */ tqp_vector->tx_group.coal.gl_adapt_enable = 1; tqp_vector->rx_group.coal.gl_adapt_enable = 1; @@ -363,18 +386,6 @@ static int hns3_nic_net_up(struct net_device *netdev) if (ret) return ret; - /* the device can work without cpu rmap, only aRFS needs it */ - ret = hns3_set_rx_cpu_rmap(netdev); - if (ret) - netdev_warn(netdev, "set rx cpu rmap fail, ret=%d!\n", ret); - - /* get irq resource for all vectors */ - ret = hns3_nic_init_irq(priv); - if (ret) { - netdev_err(netdev, "init irq failed! ret=%d\n", ret); - goto free_rmap; - } - clear_bit(HNS3_NIC_STATE_DOWN, &priv->state); /* enable the vectors */ @@ -387,22 +398,15 @@ static int hns3_nic_net_up(struct net_device *netdev) /* start the ae_dev */ ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0; - if (ret) - goto out_start_err; - - return 0; - -out_start_err: - set_bit(HNS3_NIC_STATE_DOWN, &priv->state); - while (j--) - hns3_tqp_disable(h->kinfo.tqp[j]); + if (ret) { + set_bit(HNS3_NIC_STATE_DOWN, &priv->state); + while (j--) + hns3_tqp_disable(h->kinfo.tqp[j]); - for (j = i - 1; j >= 0; j--) - hns3_vector_disable(&priv->tqp_vector[j]); + for (j = i - 1; j >= 0; j--) + hns3_vector_disable(&priv->tqp_vector[j]); + } - hns3_nic_uninit_irq(priv); -free_rmap: - hns3_free_rx_cpu_rmap(netdev); return ret; } @@ -459,6 +463,9 @@ static int hns3_nic_net_open(struct net_device *netdev) h->ae_algo->ops->set_timer_task(priv->ae_handle, true); hns3_config_xps(priv); + + netif_dbg(h, drv, netdev, "net open\n"); + return 0; } @@ -471,7 +478,7 @@ static void hns3_reset_tx_queue(struct hnae3_handle *h) for (i = 0; i < h->kinfo.num_tqps; i++) { dev_queue = netdev_get_tx_queue(ndev, - priv->ring_data[i].queue_index); + priv->ring[i].queue_index); netdev_tx_reset_queue(dev_queue); } } @@ -496,11 +503,6 @@ static void hns3_nic_net_down(struct net_device *netdev) if (ops->stop) ops->stop(priv->ae_handle); - hns3_free_rx_cpu_rmap(netdev); - - /* free irq resources */ - hns3_nic_uninit_irq(priv); - /* delay ring buffer clearing to hns3_reset_notify_uninit_enet * during reset process, because driver may not be able * to disable the ring through firmware when downing the netdev. @@ -519,6 +521,8 @@ static int hns3_nic_net_stop(struct net_device *netdev) if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state)) return 0; + netif_dbg(h, drv, netdev, "net stop\n"); + if (h->ae_algo->ops->set_timer_task) h->ae_algo->ops->set_timer_task(priv->ae_handle, false); @@ -667,7 +671,7 @@ static int hns3_set_tso(struct sk_buff *skb, u32 *paylen, return 0; ret = skb_cow_head(skb, 0); - if (unlikely(ret)) + if (unlikely(ret < 0)) return ret; l3.hdr = skb_network_header(skb); @@ -720,6 +724,8 @@ static int hns3_set_tso(struct sk_buff *skb, u32 *paylen, /* get MSS for TSO */ *mss = skb_shinfo(skb)->gso_size; + trace_hns3_tso(skb); + return 0; } @@ -948,24 +954,16 @@ static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, return 0; } -static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end) -{ - /* Config bd buffer end */ - if (!!frag_end) - hns3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, 1U); - hns3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1U); -} - -static int hns3_fill_desc_vtags(struct sk_buff *skb, - struct hns3_enet_ring *tx_ring, - u32 *inner_vlan_flag, - u32 *out_vlan_flag, - u16 *inner_vtag, - u16 *out_vtag) +static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring, + struct sk_buff *skb) { -#define HNS3_TX_VLAN_PRIO_SHIFT 13 - struct hnae3_handle *handle = tx_ring->tqp->handle; + struct vlan_ethhdr *vhdr; + int rc; + + if (!(skb->protocol == htons(ETH_P_8021Q) || + skb_vlan_tag_present(skb))) + return 0; /* Since HW limitation, if port based insert VLAN enabled, only one VLAN * header is allowed in skb, otherwise it will cause RAS error. @@ -976,8 +974,7 @@ static int hns3_fill_desc_vtags(struct sk_buff *skb, return -EINVAL; if (skb->protocol == htons(ETH_P_8021Q) && - !(tx_ring->tqp->handle->kinfo.netdev->features & - NETIF_F_HW_VLAN_CTAG_TX)) { + !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { /* When HW VLAN acceleration is turned off, and the stack * sets the protocol to 802.1q, the driver just need to * set the protocol to the encapsulated ethertype. @@ -987,130 +984,155 @@ static int hns3_fill_desc_vtags(struct sk_buff *skb, } if (skb_vlan_tag_present(skb)) { - u16 vlan_tag; - - vlan_tag = skb_vlan_tag_get(skb); - vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT; - /* Based on hw strategy, use out_vtag in two layer tag case, * and use inner_vtag in one tag case. */ - if (skb->protocol == htons(ETH_P_8021Q)) { - if (handle->port_base_vlan_state == - HNAE3_PORT_BASE_VLAN_DISABLE){ - hns3_set_field(*out_vlan_flag, - HNS3_TXD_OVLAN_B, 1); - *out_vtag = vlan_tag; - } else { - hns3_set_field(*inner_vlan_flag, - HNS3_TXD_VLAN_B, 1); - *inner_vtag = vlan_tag; - } - } else { - hns3_set_field(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1); - *inner_vtag = vlan_tag; - } - } else if (skb->protocol == htons(ETH_P_8021Q)) { - struct vlan_ethhdr *vhdr; - int rc; + if (skb->protocol == htons(ETH_P_8021Q) && + handle->port_base_vlan_state == + HNAE3_PORT_BASE_VLAN_DISABLE) + rc = HNS3_OUTER_VLAN_TAG; + else + rc = HNS3_INNER_VLAN_TAG; - rc = skb_cow_head(skb, 0); - if (unlikely(rc < 0)) - return rc; - vhdr = (struct vlan_ethhdr *)skb->data; - vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7) - << HNS3_TX_VLAN_PRIO_SHIFT); + skb->protocol = vlan_get_protocol(skb); + return rc; } + rc = skb_cow_head(skb, 0); + if (unlikely(rc < 0)) + return rc; + + vhdr = (struct vlan_ethhdr *)skb->data; + vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT) + & VLAN_PRIO_MASK); + skb->protocol = vlan_get_protocol(skb); return 0; } +static int hns3_fill_skb_desc(struct hns3_enet_ring *ring, + struct sk_buff *skb, struct hns3_desc *desc) +{ + u32 ol_type_vlan_len_msec = 0; + u32 type_cs_vlan_tso = 0; + u32 paylen = skb->len; + u16 inner_vtag = 0; + u16 out_vtag = 0; + u16 mss = 0; + int ret; + + ret = hns3_handle_vtags(ring, skb); + if (unlikely(ret < 0)) { + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_vlan_err++; + u64_stats_update_end(&ring->syncp); + return ret; + } else if (ret == HNS3_INNER_VLAN_TAG) { + inner_vtag = skb_vlan_tag_get(skb); + inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & + VLAN_PRIO_MASK; + hns3_set_field(type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1); + } else if (ret == HNS3_OUTER_VLAN_TAG) { + out_vtag = skb_vlan_tag_get(skb); + out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & + VLAN_PRIO_MASK; + hns3_set_field(ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B, + 1); + } + + if (skb->ip_summed == CHECKSUM_PARTIAL) { + u8 ol4_proto, il4_proto; + + skb_reset_mac_len(skb); + + ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto); + if (unlikely(ret < 0)) { + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_l4_proto_err++; + u64_stats_update_end(&ring->syncp); + return ret; + } + + ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto, + &type_cs_vlan_tso, + &ol_type_vlan_len_msec); + if (unlikely(ret < 0)) { + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_l2l3l4_err++; + u64_stats_update_end(&ring->syncp); + return ret; + } + + ret = hns3_set_tso(skb, &paylen, &mss, + &type_cs_vlan_tso); + if (unlikely(ret < 0)) { + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_tso_err++; + u64_stats_update_end(&ring->syncp); + return ret; + } + } + + /* Set txbd */ + desc->tx.ol_type_vlan_len_msec = + cpu_to_le32(ol_type_vlan_len_msec); + desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso); + desc->tx.paylen = cpu_to_le32(paylen); + desc->tx.mss = cpu_to_le16(mss); + desc->tx.vlan_tag = cpu_to_le16(inner_vtag); + desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag); + + return 0; +} + static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv, - unsigned int size, int frag_end, - enum hns_desc_type type) + unsigned int size, enum hns_desc_type type) { +#define HNS3_LIKELY_BD_NUM 1 + struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; struct hns3_desc *desc = &ring->desc[ring->next_to_use]; struct device *dev = ring_to_dev(ring); - struct skb_frag_struct *frag; + skb_frag_t *frag; unsigned int frag_buf_num; int k, sizeoflast; dma_addr_t dma; if (type == DESC_TYPE_SKB) { struct sk_buff *skb = (struct sk_buff *)priv; - u32 ol_type_vlan_len_msec = 0; - u32 type_cs_vlan_tso = 0; - u32 paylen = skb->len; - u16 inner_vtag = 0; - u16 out_vtag = 0; - u16 mss = 0; int ret; - ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso, - &ol_type_vlan_len_msec, - &inner_vtag, &out_vtag); - if (unlikely(ret)) + ret = hns3_fill_skb_desc(ring, skb, desc); + if (unlikely(ret < 0)) return ret; - if (skb->ip_summed == CHECKSUM_PARTIAL) { - u8 ol4_proto, il4_proto; - - skb_reset_mac_len(skb); - - ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto); - if (unlikely(ret)) - return ret; - - ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto, - &type_cs_vlan_tso, - &ol_type_vlan_len_msec); - if (unlikely(ret)) - return ret; - - ret = hns3_set_tso(skb, &paylen, &mss, - &type_cs_vlan_tso); - if (unlikely(ret)) - return ret; - } - - /* Set txbd */ - desc->tx.ol_type_vlan_len_msec = - cpu_to_le32(ol_type_vlan_len_msec); - desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso); - desc->tx.paylen = cpu_to_le32(paylen); - desc->tx.mss = cpu_to_le16(mss); - desc->tx.vlan_tag = cpu_to_le16(inner_vtag); - desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag); - dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); } else { - frag = (struct skb_frag_struct *)priv; + frag = (skb_frag_t *)priv; dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); } if (unlikely(dma_mapping_error(dev, dma))) { + u64_stats_update_begin(&ring->syncp); ring->stats.sw_err_cnt++; + u64_stats_update_end(&ring->syncp); return -ENOMEM; } desc_cb->length = size; if (likely(size <= HNS3_MAX_BD_SIZE)) { - u16 bdtp_fe_sc_vld_ra_ri = 0; - desc_cb->priv = priv; desc_cb->dma = dma; desc_cb->type = type; desc->addr = cpu_to_le64(dma); desc->tx.send_size = cpu_to_le16(size); - hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end); desc->tx.bdtp_fe_sc_vld_ra_ri = - cpu_to_le16(bdtp_fe_sc_vld_ra_ri); + cpu_to_le16(BIT(HNS3_TXD_VLD_B)); + trace_hns3_tx_desc(ring, ring->next_to_use); ring_ptr_move_fw(ring, next_to_use); - return 0; + return HNS3_LIKELY_BD_NUM; } frag_buf_num = hns3_tx_bd_count(size); @@ -1119,8 +1141,6 @@ static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv, /* When frag size is bigger than hardware limit, split this frag */ for (k = 0; k < frag_buf_num; k++) { - u16 bdtp_fe_sc_vld_ra_ri = 0; - /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */ desc_cb->priv = priv; desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k; @@ -1131,12 +1151,10 @@ static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv, desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k); desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ? (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE); - hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, - frag_end && (k == frag_buf_num - 1) ? - 1 : 0); desc->tx.bdtp_fe_sc_vld_ra_ri = - cpu_to_le16(bdtp_fe_sc_vld_ra_ri); + cpu_to_le16(BIT(HNS3_TXD_VLD_B)); + trace_hns3_tx_desc(ring, ring->next_to_use); /* move ring pointer to next */ ring_ptr_move_fw(ring, next_to_use); @@ -1144,31 +1162,78 @@ static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv, desc = &ring->desc[ring->next_to_use]; } - return 0; + return frag_buf_num; } -static int hns3_nic_bd_num(struct sk_buff *skb) +static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size, + unsigned int bd_num) { - int size = skb_headlen(skb); - int i, bd_num; + unsigned int size; + int i; - /* if the total len is within the max bd limit */ - if (likely(skb->len <= HNS3_MAX_BD_SIZE)) - return skb_shinfo(skb)->nr_frags + 1; + size = skb_headlen(skb); + while (size > HNS3_MAX_BD_SIZE) { + bd_size[bd_num++] = HNS3_MAX_BD_SIZE; + size -= HNS3_MAX_BD_SIZE; - bd_num = hns3_tx_bd_count(size); + if (bd_num > HNS3_MAX_TSO_BD_NUM) + return bd_num; + } - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; - int frag_bd_num; + if (size) { + bd_size[bd_num++] = size; + if (bd_num > HNS3_MAX_TSO_BD_NUM) + return bd_num; + } + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; size = skb_frag_size(frag); - frag_bd_num = hns3_tx_bd_count(size); + if (!size) + continue; - if (unlikely(frag_bd_num > HNS3_MAX_BD_PER_FRAG)) - return -ENOMEM; + while (size > HNS3_MAX_BD_SIZE) { + bd_size[bd_num++] = HNS3_MAX_BD_SIZE; + size -= HNS3_MAX_BD_SIZE; + + if (bd_num > HNS3_MAX_TSO_BD_NUM) + return bd_num; + } + + bd_size[bd_num++] = size; + if (bd_num > HNS3_MAX_TSO_BD_NUM) + return bd_num; + } + + return bd_num; +} + +static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size) +{ + struct sk_buff *frag_skb; + unsigned int bd_num = 0; + + /* If the total len is within the max bd limit */ + if (likely(skb->len <= HNS3_MAX_BD_SIZE && !skb_has_frag_list(skb) && + skb_shinfo(skb)->nr_frags < HNS3_MAX_NON_TSO_BD_NUM)) + return skb_shinfo(skb)->nr_frags + 1U; + + /* The below case will always be linearized, return + * HNS3_MAX_BD_NUM_TSO + 1U to make sure it is linearized. + */ + if (unlikely(skb->len > HNS3_MAX_TSO_SIZE || + (!skb_is_gso(skb) && skb->len > HNS3_MAX_NON_TSO_SIZE))) + return HNS3_MAX_TSO_BD_NUM + 1U; + + bd_num = hns3_skb_bd_num(skb, bd_size, bd_num); + + if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM) + return bd_num; - bd_num += frag_bd_num; + skb_walk_frags(skb, frag_skb) { + bd_num = hns3_skb_bd_num(frag_skb, bd_size, bd_num); + if (bd_num > HNS3_MAX_TSO_BD_NUM) + return bd_num; } return bd_num; @@ -1187,26 +1252,26 @@ static unsigned int hns3_gso_hdr_len(struct sk_buff *skb) * 7 frags to to be larger than gso header len + mss, and the remaining * continuous 7 frags to be larger than MSS except the last 7 frags. */ -static bool hns3_skb_need_linearized(struct sk_buff *skb) +static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size, + unsigned int bd_num) { - int bd_limit = HNS3_MAX_BD_PER_FRAG - 1; unsigned int tot_len = 0; int i; - for (i = 0; i < bd_limit; i++) - tot_len += skb_frag_size(&skb_shinfo(skb)->frags[i]); + for (i = 0; i < HNS3_MAX_NON_TSO_BD_NUM - 1U; i++) + tot_len += bd_size[i]; - /* ensure headlen + the first 7 frags is greater than mss + header - * and the first 7 frags is greater than mss. - */ - if (((tot_len + skb_headlen(skb)) < (skb_shinfo(skb)->gso_size + - hns3_gso_hdr_len(skb))) || (tot_len < skb_shinfo(skb)->gso_size)) + /* ensure the first 8 frags is greater than mss + header */ + if (tot_len + bd_size[HNS3_MAX_NON_TSO_BD_NUM - 1U] < + skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb)) return true; - /* ensure the remaining continuous 7 buffer is greater than mss */ - for (i = 0; i < (skb_shinfo(skb)->nr_frags - bd_limit - 1); i++) { - tot_len -= skb_frag_size(&skb_shinfo(skb)->frags[i]); - tot_len += skb_frag_size(&skb_shinfo(skb)->frags[i + bd_limit]); + /* ensure every continuous 7 buffer is greater than mss + * except the last one. + */ + for (i = 0; i < bd_num - HNS3_MAX_NON_TSO_BD_NUM; i++) { + tot_len -= bd_size[i]; + tot_len += bd_size[i + HNS3_MAX_NON_TSO_BD_NUM - 1U]; if (tot_len < skb_shinfo(skb)->gso_size) return true; @@ -1215,31 +1280,40 @@ static bool hns3_skb_need_linearized(struct sk_buff *skb) return false; } -static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, - struct sk_buff **out_skb) +void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size) { - struct sk_buff *skb = *out_skb; - int bd_num; - - bd_num = hns3_nic_bd_num(skb); - if (bd_num < 0) - return bd_num; + int i = 0; - if (unlikely(bd_num > HNS3_MAX_BD_PER_FRAG)) { - struct sk_buff *new_skb; + for (i = 0; i < MAX_SKB_FRAGS; i++) + size[i] = skb_frag_size(&shinfo->frags[i]); +} - if (skb_is_gso(skb) && !hns3_skb_need_linearized(skb)) +static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, + struct net_device *netdev, + struct sk_buff *skb) +{ + struct hns3_nic_priv *priv = netdev_priv(netdev); + unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U]; + unsigned int bd_num; + + bd_num = hns3_tx_bd_num(skb, bd_size); + if (unlikely(bd_num > HNS3_MAX_NON_TSO_BD_NUM)) { + if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) && + !hns3_skb_need_linearized(skb, bd_size, bd_num)) { + trace_hns3_over_8bd(skb); goto out; + } + + if (__skb_linearize(skb)) + return -ENOMEM; bd_num = hns3_tx_bd_count(skb->len); - if (unlikely(ring_space(ring) < bd_num)) - return -EBUSY; - /* manual split the send packet */ - new_skb = skb_copy(skb, GFP_ATOMIC); - if (!new_skb) + if ((skb_is_gso(skb) && bd_num > HNS3_MAX_TSO_BD_NUM) || + (!skb_is_gso(skb) && + bd_num > HNS3_MAX_NON_TSO_BD_NUM)) { + trace_hns3_over_8bd(skb); return -ENOMEM; - dev_kfree_skb_any(skb); - *out_skb = new_skb; + } u64_stats_update_begin(&ring->syncp); ring->stats.tx_copy++; @@ -1247,10 +1321,23 @@ static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, } out: - if (unlikely(ring_space(ring) < bd_num)) - return -EBUSY; + if (likely(ring_space(ring) >= bd_num)) + return bd_num; - return bd_num; + netif_stop_subqueue(netdev, ring->queue_index); + smp_mb(); /* Memory barrier before checking ring_space */ + + /* Start queue in case hns3_clean_tx_ring has just made room + * available and has not seen the queue stopped state performed + * by netif_stop_subqueue above. + */ + if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) && + !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) { + netif_start_subqueue(netdev, ring->queue_index); + return bd_num; + } + + return -EBUSY; } static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig) @@ -1283,75 +1370,103 @@ static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig) } } +static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring, + struct sk_buff *skb, enum hns_desc_type type) +{ + unsigned int size = skb_headlen(skb); + int i, ret, bd_num = 0; + + if (size) { + ret = hns3_fill_desc(ring, skb, size, type); + if (unlikely(ret < 0)) + return ret; + + bd_num += ret; + } + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + + size = skb_frag_size(frag); + if (!size) + continue; + + ret = hns3_fill_desc(ring, frag, size, DESC_TYPE_PAGE); + if (unlikely(ret < 0)) + return ret; + + bd_num += ret; + } + + return bd_num; +} + netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); - struct hns3_nic_ring_data *ring_data = - &tx_ring_data(priv, skb->queue_mapping); - struct hns3_enet_ring *ring = ring_data->ring; + struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping]; struct netdev_queue *dev_queue; - struct skb_frag_struct *frag; - int next_to_use_head; - int buf_num; - int seg_num; - int size; + int pre_ntu, next_to_use_head; + struct sk_buff *frag_skb; + int bd_num = 0; int ret; - int i; + + /* Hardware can only handle short frames above 32 bytes */ + if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) + return NETDEV_TX_OK; /* Prefetch the data used later */ prefetch(skb->data); - buf_num = hns3_nic_maybe_stop_tx(ring, &skb); - if (unlikely(buf_num <= 0)) { - if (buf_num == -EBUSY) { + ret = hns3_nic_maybe_stop_tx(ring, netdev, skb); + if (unlikely(ret <= 0)) { + if (ret == -EBUSY) { u64_stats_update_begin(&ring->syncp); ring->stats.tx_busy++; u64_stats_update_end(&ring->syncp); - goto out_net_tx_busy; - } else if (buf_num == -ENOMEM) { + return NETDEV_TX_BUSY; + } else if (ret == -ENOMEM) { u64_stats_update_begin(&ring->syncp); ring->stats.sw_err_cnt++; u64_stats_update_end(&ring->syncp); } - if (net_ratelimit()) - netdev_err(netdev, "xmit error: %d!\n", buf_num); - + hns3_rl_err(netdev, "xmit error: %d!\n", ret); goto out_err_tx_ok; } - /* No. of segments (plus a header) */ - seg_num = skb_shinfo(skb)->nr_frags + 1; - /* Fill the first part */ - size = skb_headlen(skb); - next_to_use_head = ring->next_to_use; - ret = hns3_fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0, - DESC_TYPE_SKB); - if (unlikely(ret)) + ret = hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB); + if (unlikely(ret < 0)) goto fill_err; - /* Fill the fragments */ - for (i = 1; i < seg_num; i++) { - frag = &skb_shinfo(skb)->frags[i - 1]; - size = skb_frag_size(frag); + bd_num += ret; - ret = hns3_fill_desc(ring, frag, size, - seg_num - 1 == i ? 1 : 0, - DESC_TYPE_PAGE); + if (!skb_has_frag_list(skb)) + goto out; - if (unlikely(ret)) + skb_walk_frags(skb, frag_skb) { + ret = hns3_fill_skb_to_desc(ring, frag_skb, DESC_TYPE_PAGE); + if (unlikely(ret < 0)) goto fill_err; + + bd_num += ret; } +out: + pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) : + (ring->desc_num - 1); + ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |= + cpu_to_le16(BIT(HNS3_TXD_FE_B)); + trace_hns3_tx_desc(ring, pre_ntu); /* Complete translate all packets */ - dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index); + dev_queue = netdev_get_tx_queue(netdev, ring->queue_index); netdev_tx_sent_queue(dev_queue, skb->len); wmb(); /* Commit all data before submit */ - hnae3_queue_xmit(ring->tqp, buf_num); + hnae3_queue_xmit(ring->tqp, bd_num); return NETDEV_TX_OK; @@ -1361,12 +1476,6 @@ fill_err: out_err_tx_ok: dev_kfree_skb_any(skb); return NETDEV_TX_OK; - -out_net_tx_busy: - netif_stop_subqueue(netdev, ring_data->queue_index); - smp_mb(); /* Commit all data before submit */ - - return NETDEV_TX_BUSY; } static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p) @@ -1384,6 +1493,16 @@ static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p) return 0; } + /* For VF device, if there is a perm_addr, then the user will not + * be allowed to change the address. + */ + if (!hns3_is_phys_func(h->pdev) && + !is_zero_ether_addr(netdev->perm_addr)) { + netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n", + netdev->perm_addr, mac_addr->sa_data); + return -EPERM; + } + ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false); if (ret) { netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret); @@ -1448,6 +1567,37 @@ static int hns3_nic_set_features(struct net_device *netdev, return 0; } +static netdev_features_t hns3_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ +#define HNS3_MAX_HDR_LEN 480U +#define HNS3_MAX_L4_HDR_LEN 60U + + size_t len; + + if (skb->ip_summed != CHECKSUM_PARTIAL) + return features; + + if (skb->encapsulation) + len = skb_inner_transport_header(skb) - skb->data; + else + len = skb_transport_header(skb) - skb->data; + + /* Assume L4 is 60 byte as TCP is the only protocol with a + * a flexible value, and it's max len is 60 bytes. + */ + len += HNS3_MAX_L4_HDR_LEN; + + /* Hardware only supports checksum on the skb with a max header + * len of 480 bytes. + */ + if (len > HNS3_MAX_HDR_LEN) + features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); + + return features; +} + static void hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { @@ -1476,17 +1626,25 @@ static void hns3_nic_get_stats64(struct net_device *netdev, for (idx = 0; idx < queue_num; idx++) { /* fetch the tx stats */ - ring = priv->ring_data[idx].ring; + ring = &priv->ring[idx]; do { start = u64_stats_fetch_begin_irq(&ring->syncp); tx_bytes += ring->stats.tx_bytes; tx_pkts += ring->stats.tx_pkts; tx_drop += ring->stats.sw_err_cnt; + tx_drop += ring->stats.tx_vlan_err; + tx_drop += ring->stats.tx_l4_proto_err; + tx_drop += ring->stats.tx_l2l3l4_err; + tx_drop += ring->stats.tx_tso_err; tx_errors += ring->stats.sw_err_cnt; + tx_errors += ring->stats.tx_vlan_err; + tx_errors += ring->stats.tx_l4_proto_err; + tx_errors += ring->stats.tx_l2l3l4_err; + tx_errors += ring->stats.tx_tso_err; } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); /* fetch the rx stats */ - ring = priv->ring_data[idx + queue_num].ring; + ring = &priv->ring[idx + queue_num]; do { start = u64_stats_fetch_begin_irq(&ring->syncp); rx_bytes += ring->stats.rx_bytes; @@ -1550,6 +1708,8 @@ static int hns3_setup_tc(struct net_device *netdev, void *type_data) h = hns3_get_handle(netdev); kinfo = &h->kinfo; + netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc); + return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ? kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP; } @@ -1593,6 +1753,10 @@ static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, struct hnae3_handle *h = hns3_get_handle(netdev); int ret = -EIO; + netif_dbg(h, drv, netdev, + "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n", + vf, vlan, qos, ntohs(vlan_proto)); + if (h->ae_algo->ops->set_vf_vlan_filter) ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan, qos, vlan_proto); @@ -1600,6 +1764,29 @@ static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, return ret; } +static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable) +{ + struct hnae3_handle *handle = hns3_get_handle(netdev); + + if (hns3_nic_resetting(netdev)) + return -EBUSY; + + if (!handle->ae_algo->ops->set_vf_spoofchk) + return -EOPNOTSUPP; + + return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable); +} + +static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable) +{ + struct hnae3_handle *handle = hns3_get_handle(netdev); + + if (!handle->ae_algo->ops->set_vf_trust) + return -EOPNOTSUPP; + + return handle->ae_algo->ops->set_vf_trust(handle, vf, enable); +} + static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu) { struct hnae3_handle *h = hns3_get_handle(netdev); @@ -1611,6 +1798,9 @@ static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu) if (!h->ae_algo->ops->set_mtu) return -EOPNOTSUPP; + netif_dbg(h, drv, netdev, + "change mtu from %u to %d\n", netdev->mtu, new_mtu); + ret = h->ae_algo->ops->set_mtu(h, new_mtu); if (ret) netdev_err(netdev, "failed to change MTU in hardware %d\n", @@ -1625,7 +1815,7 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) { struct hns3_nic_priv *priv = netdev_priv(ndev); struct hnae3_handle *h = hns3_get_handle(ndev); - struct hns3_enet_ring *tx_ring = NULL; + struct hns3_enet_ring *tx_ring; struct napi_struct *napi; int timeout_queue = 0; int hw_head, hw_tail; @@ -1646,6 +1836,9 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) time_after(jiffies, (trans_start + ndev->watchdog_timeo))) { timeout_queue = i; + netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n", + q->state, + jiffies_to_msecs(jiffies - trans_start)); break; } } @@ -1659,7 +1852,7 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) priv->tx_timeout_count++; - tx_ring = priv->ring_data[timeout_queue].ring; + tx_ring = &priv->ring[timeout_queue]; napi = &tx_ring->tqp_vector->napi; netdev_info(ndev, @@ -1680,15 +1873,12 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) /* When mac received many pause frames continuous, it's unable to send * packets, which may cause tx timeout */ - if (h->ae_algo->ops->update_stats && - h->ae_algo->ops->get_mac_pause_stats) { - u64 tx_pause_cnt, rx_pause_cnt; + if (h->ae_algo->ops->get_mac_stats) { + struct hns3_mac_stats mac_stats; - h->ae_algo->ops->update_stats(h, &ndev->stats); - h->ae_algo->ops->get_mac_pause_stats(h, &tx_pause_cnt, - &rx_pause_cnt); + h->ae_algo->ops->get_mac_stats(h, &mac_stats); netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n", - tx_pause_cnt, rx_pause_cnt); + mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt); } hw_head = readl_relaxed(tx_ring->tqp->io_base + @@ -1721,7 +1911,7 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) return true; } -static void hns3_nic_net_timeout(struct net_device *ndev) +static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue) { struct hns3_nic_priv *priv = netdev_priv(ndev); struct hnae3_handle *h = priv->ae_handle; @@ -1762,6 +1952,57 @@ static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, } #endif +static int hns3_nic_get_vf_config(struct net_device *ndev, int vf, + struct ifla_vf_info *ivf) +{ + struct hnae3_handle *h = hns3_get_handle(ndev); + + if (!h->ae_algo->ops->get_vf_config) + return -EOPNOTSUPP; + + return h->ae_algo->ops->get_vf_config(h, vf, ivf); +} + +static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf, + int link_state) +{ + struct hnae3_handle *h = hns3_get_handle(ndev); + + if (!h->ae_algo->ops->set_vf_link_state) + return -EOPNOTSUPP; + + return h->ae_algo->ops->set_vf_link_state(h, vf, link_state); +} + +static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf, + int min_tx_rate, int max_tx_rate) +{ + struct hnae3_handle *h = hns3_get_handle(ndev); + + if (!h->ae_algo->ops->set_vf_rate) + return -EOPNOTSUPP; + + return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate, + false); +} + +static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) +{ + struct hnae3_handle *h = hns3_get_handle(netdev); + + if (!h->ae_algo->ops->set_vf_mac) + return -EOPNOTSUPP; + + if (is_multicast_ether_addr(mac)) { + netdev_err(netdev, + "Invalid MAC:%pM specified. Could not set MAC\n", + mac); + return -EINVAL; + } + + return h->ae_algo->ops->set_vf_mac(h, vf_id, mac); +} + static const struct net_device_ops hns3_nic_netdev_ops = { .ndo_open = hns3_nic_net_open, .ndo_stop = hns3_nic_net_stop, @@ -1771,16 +2012,22 @@ static const struct net_device_ops hns3_nic_netdev_ops = { .ndo_do_ioctl = hns3_nic_do_ioctl, .ndo_change_mtu = hns3_nic_change_mtu, .ndo_set_features = hns3_nic_set_features, + .ndo_features_check = hns3_features_check, .ndo_get_stats64 = hns3_nic_get_stats64, .ndo_setup_tc = hns3_nic_setup_tc, .ndo_set_rx_mode = hns3_nic_set_rx_mode, .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid, .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan, + .ndo_set_vf_spoofchk = hns3_set_vf_spoofchk, + .ndo_set_vf_trust = hns3_set_vf_trust, #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = hns3_rx_flow_steer, #endif - + .ndo_get_vf_config = hns3_nic_get_vf_config, + .ndo_set_vf_link_state = hns3_nic_set_vf_link_state, + .ndo_set_vf_rate = hns3_nic_set_vf_rate, + .ndo_set_vf_mac = hns3_nic_set_vf_mac, }; bool hns3_is_phys_func(struct pci_dev *pdev) @@ -1800,7 +2047,7 @@ bool hns3_is_phys_func(struct pci_dev *pdev) case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF: return false; default: - dev_warn(&pdev->dev, "un-recognized pci device-id %d", + dev_warn(&pdev->dev, "un-recognized pci device-id %u", dev_id); } @@ -1847,10 +2094,8 @@ static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL); - if (!ae_dev) { - ret = -ENOMEM; - return ret; - } + if (!ae_dev) + return -ENOMEM; ae_dev->pdev = pdev; ae_dev->flag = ent->driver_data; @@ -1963,7 +2208,8 @@ static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev) ops = ae_dev->ops; /* request the reset */ - if (ops->reset_event) { + if (ops->reset_event && ops->get_reset_level && + ops->set_default_reset_request) { if (ae_dev->hw_err_reset_req) { reset_type = ops->get_reset_level(ae_dev, &ae_dev->hw_err_reset_req); @@ -2025,9 +2271,8 @@ static void hns3_set_default_feature(struct net_device *netdev) NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | - NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC; - - netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; + NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC | + NETIF_F_TSO_MANGLEID | NETIF_F_FRAGLIST; netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; @@ -2037,21 +2282,24 @@ static void hns3_set_default_feature(struct net_device *netdev) NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | - NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC; + NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC | + NETIF_F_FRAGLIST; netdev->vlan_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | - NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC; + NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC | + NETIF_F_FRAGLIST; netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | - NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC; + NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC | + NETIF_F_FRAGLIST; if (pdev->revision >= 0x21) { netdev->hw_features |= NETIF_F_GRO_HW; @@ -2067,7 +2315,7 @@ static void hns3_set_default_feature(struct net_device *netdev) static int hns3_alloc_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb) { - unsigned int order = hnae3_page_order(ring); + unsigned int order = hns3_page_order(ring); struct page *p; p = dev_alloc_pages(order); @@ -2078,7 +2326,7 @@ static int hns3_alloc_buffer(struct hns3_enet_ring *ring, cb->page_offset = 0; cb->reuse_flag = 0; cb->buf = page_address(p); - cb->length = hnae3_page_size(ring); + cb->length = hns3_page_size(ring); cb->type = DESC_TYPE_PAGE; return 0; @@ -2276,21 +2524,22 @@ static int is_valid_clean_head(struct hns3_enet_ring *ring, int h) void hns3_clean_tx_ring(struct hns3_enet_ring *ring) { - struct net_device *netdev = ring->tqp->handle->kinfo.netdev; + struct net_device *netdev = ring_to_netdev(ring); struct hns3_nic_priv *priv = netdev_priv(netdev); struct netdev_queue *dev_queue; int bytes, pkts; int head; head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG); - rmb(); /* Make sure head is ready before touch any data */ if (is_ring_empty(ring) || head == ring->next_to_clean) return; /* no data to poll */ + rmb(); /* Make sure head is ready before touch any data */ + if (unlikely(!is_valid_clean_head(ring, head))) { - netdev_err(netdev, "wrong head (%d, %d-%d)\n", head, - ring->next_to_use, ring->next_to_clean); + hns3_rl_err(netdev, "wrong head (%d, %d-%d)\n", head, + ring->next_to_use, ring->next_to_clean); u64_stats_update_begin(&ring->syncp); ring->stats.io_err_cnt++; @@ -2313,8 +2562,8 @@ void hns3_clean_tx_ring(struct hns3_enet_ring *ring) dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index); netdev_tx_completed_queue(dev_queue, pkts, bytes); - if (unlikely(pkts && netif_carrier_ok(netdev) && - (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) { + if (unlikely(netif_carrier_ok(netdev) && + ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) { /* Make sure that anybody stopping the queue after this * sees the new next_to_clean. */ @@ -2357,8 +2606,9 @@ static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, ring->stats.sw_err_cnt++; u64_stats_update_end(&ring->syncp); - netdev_err(ring->tqp->handle->kinfo.netdev, - "hnae reserve buffer map failed.\n"); + hns3_rl_err(ring_to_netdev(ring), + "alloc rx buffer failed: %d\n", + ret); break; } hns3_replace_buffer(ring, ring->next_to_use, &res_cbs); @@ -2375,13 +2625,19 @@ static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG); } +static bool hns3_page_is_reusable(struct page *page) +{ + return page_to_nid(page) == numa_mem_id() && + !page_is_pfmemalloc(page); +} + static void hns3_nic_reuse_page(struct sk_buff *skb, int i, struct hns3_enet_ring *ring, int pull_len, struct hns3_desc_cb *desc_cb) { struct hns3_desc *desc = &ring->desc[ring->next_to_clean]; int size = le16_to_cpu(desc->rx.size); - u32 truesize = hnae3_buf_size(ring); + u32 truesize = hns3_buf_size(ring); skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len, size - pull_len, truesize); @@ -2389,14 +2645,14 @@ static void hns3_nic_reuse_page(struct sk_buff *skb, int i, /* Avoid re-using remote pages, or the stack is still using the page * when page_offset rollback to zero, flag default unreuse */ - if (unlikely(page_to_nid(desc_cb->priv) != numa_mem_id()) || + if (unlikely(!hns3_page_is_reusable(desc_cb->priv)) || (!desc_cb->page_offset && page_count(desc_cb->priv) > 1)) return; /* Move offset up to the next cache line */ desc_cb->page_offset += truesize; - if (desc_cb->page_offset + truesize <= hnae3_page_size(ring)) { + if (desc_cb->page_offset + truesize <= hns3_page_size(ring)) { desc_cb->reuse_flag = 1; /* Bump ref count on page before it is given */ get_page(desc_cb->priv); @@ -2443,9 +2699,9 @@ static int hns3_gro_complete(struct sk_buff *skb, u32 l234info) th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr, &iph->daddr, 0); } else { - netdev_err(skb->dev, - "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n", - be16_to_cpu(type), depth); + hns3_rl_err(skb->dev, + "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n", + be16_to_cpu(type), depth); return -EFAULT; } @@ -2459,13 +2715,16 @@ static int hns3_gro_complete(struct sk_buff *skb, u32 l234info) skb->csum_start = (unsigned char *)th - skb->head; skb->csum_offset = offsetof(struct tcphdr, check); skb->ip_summed = CHECKSUM_PARTIAL; + + trace_hns3_gro(skb); + return 0; } static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, u32 l234info, u32 bd_base_info, u32 ol_info) { - struct net_device *netdev = ring->tqp->handle->kinfo.netdev; + struct net_device *netdev = ring_to_netdev(ring); int l3_type, l4_type; int ol4_type; @@ -2579,15 +2838,14 @@ static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring, static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, unsigned char *va) { -#define HNS3_NEED_ADD_FRAG 1 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean]; - struct net_device *netdev = ring->tqp->handle->kinfo.netdev; + struct net_device *netdev = ring_to_netdev(ring); struct sk_buff *skb; ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE); skb = ring->skb; if (unlikely(!skb)) { - netdev_err(netdev, "alloc rx skb fail\n"); + hns3_rl_err(netdev, "alloc rx skb fail\n"); u64_stats_update_begin(&ring->syncp); ring->stats.sw_err_cnt++; @@ -2596,6 +2854,7 @@ static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, return -ENOMEM; } + trace_hns3_rx_desc(ring); prefetchw(skb->data); ring->pending_buf = 1; @@ -2605,7 +2864,7 @@ static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long))); /* We can reuse buffer as-is, just make sure it is local */ - if (likely(page_to_nid(desc_cb->priv) == numa_mem_id())) + if (likely(hns3_page_is_reusable(desc_cb->priv))) desc_cb->reuse_flag = 1; else /* This page cannot be reused so discard it */ put_page(desc_cb->priv); @@ -2623,33 +2882,19 @@ static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, desc_cb); ring_ptr_move_fw(ring, next_to_clean); - return HNS3_NEED_ADD_FRAG; + return 0; } -static int hns3_add_frag(struct hns3_enet_ring *ring, struct hns3_desc *desc, - struct sk_buff **out_skb, bool pending) +static int hns3_add_frag(struct hns3_enet_ring *ring) { - struct sk_buff *skb = *out_skb; - struct sk_buff *head_skb = *out_skb; + struct sk_buff *skb = ring->skb; + struct sk_buff *head_skb = skb; struct sk_buff *new_skb; struct hns3_desc_cb *desc_cb; - struct hns3_desc *pre_desc; + struct hns3_desc *desc; u32 bd_base_info; - int pre_bd; - - /* if there is pending bd, the SW param next_to_clean has moved - * to next and the next is NULL - */ - if (pending) { - pre_bd = (ring->next_to_clean - 1 + ring->desc_num) % - ring->desc_num; - pre_desc = &ring->desc[pre_bd]; - bd_base_info = le32_to_cpu(pre_desc->rx.bd_base_info); - } else { - bd_base_info = le32_to_cpu(desc->rx.bd_base_info); - } - while (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { + do { desc = &ring->desc[ring->next_to_clean]; desc_cb = &ring->desc_cb[ring->next_to_clean]; bd_base_info = le32_to_cpu(desc->rx.bd_base_info); @@ -2659,11 +2904,10 @@ static int hns3_add_frag(struct hns3_enet_ring *ring, struct hns3_desc *desc, return -ENXIO; if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) { - new_skb = napi_alloc_skb(&ring->tqp_vector->napi, - HNS3_RX_HEAD_SIZE); + new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0); if (unlikely(!new_skb)) { - netdev_err(ring->tqp->handle->kinfo.netdev, - "alloc rx skb frag fail\n"); + hns3_rl_err(ring_to_netdev(ring), + "alloc rx fraglist skb fail\n"); return -ENXIO; } ring->frag_num = 0; @@ -2678,16 +2922,17 @@ static int hns3_add_frag(struct hns3_enet_ring *ring, struct hns3_desc *desc, } if (ring->tail_skb) { - head_skb->truesize += hnae3_buf_size(ring); + head_skb->truesize += hns3_buf_size(ring); head_skb->data_len += le16_to_cpu(desc->rx.size); head_skb->len += le16_to_cpu(desc->rx.size); skb = ring->tail_skb; } hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb); + trace_hns3_rx_desc(ring); ring_ptr_move_fw(ring, next_to_clean); ring->pending_buf++; - } + } while (!(bd_base_info & BIT(HNS3_RXD_FE_B))); return 0; } @@ -2738,7 +2983,7 @@ static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring, static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) { - struct net_device *netdev = ring->tqp->handle->kinfo.netdev; + struct net_device *netdev = ring_to_netdev(ring); enum hns3_pkt_l2t_type l2_frame_type; u32 bd_base_info, l234info, ol_info; struct hns3_desc *desc; @@ -2813,8 +3058,7 @@ static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) return 0; } -static int hns3_handle_rx_bd(struct hns3_enet_ring *ring, - struct sk_buff **out_skb) +static int hns3_handle_rx_bd(struct hns3_enet_ring *ring) { struct sk_buff *skb = ring->skb; struct hns3_desc_cb *desc_cb; @@ -2852,32 +3096,27 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring, if (!skb) { ret = hns3_alloc_skb(ring, length, ring->va); - *out_skb = skb = ring->skb; + skb = ring->skb; if (ret < 0) /* alloc buffer fail */ return ret; - if (ret > 0) { /* need add frag */ - ret = hns3_add_frag(ring, desc, &skb, false); + if (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { /* need add frag */ + ret = hns3_add_frag(ring); if (ret) return ret; - - /* As the head data may be changed when GRO enable, copy - * the head data in after other data rx completed - */ - memcpy(skb->data, ring->va, - ALIGN(ring->pull_len, sizeof(long))); } } else { - ret = hns3_add_frag(ring, desc, &skb, true); + ret = hns3_add_frag(ring); if (ret) return ret; + } - /* As the head data may be changed when GRO enable, copy - * the head data in after other data rx completed - */ + /* As the head data may be changed when GRO enable, copy + * the head data in after other data rx completed + */ + if (skb->len > HNS3_RX_HEAD_SIZE) memcpy(skb->data, ring->va, ALIGN(ring->pull_len, sizeof(long))); - } ret = hns3_handle_bdinfo(ring, skb); if (unlikely(ret)) { @@ -2886,8 +3125,6 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring, } skb_record_rx_queue(skb, ring->tqp->tqp_index); - *out_skb = skb; - return 0; } @@ -2895,56 +3132,48 @@ int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget, void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *)) { #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16 - int recv_pkts, recv_bds, clean_count, err; int unused_count = hns3_desc_unused(ring); - struct sk_buff *skb = ring->skb; - int num; + int recv_pkts = 0; + int recv_bds = 0; + int err, num; num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG); - rmb(); /* Make sure num taken effect before the other data is touched */ - - recv_pkts = 0, recv_bds = 0, clean_count = 0; num -= unused_count; unused_count -= ring->pending_buf; + if (num <= 0) + goto out; + + rmb(); /* Make sure num taken effect before the other data is touched */ + while (recv_pkts < budget && recv_bds < num) { /* Reuse or realloc buffers */ - if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) { - hns3_nic_alloc_rx_buffers(ring, - clean_count + unused_count); - clean_count = 0; + if (unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) { + hns3_nic_alloc_rx_buffers(ring, unused_count); unused_count = hns3_desc_unused(ring) - ring->pending_buf; } /* Poll one pkt */ - err = hns3_handle_rx_bd(ring, &skb); - if (unlikely(!skb)) /* This fault cannot be repaired */ + err = hns3_handle_rx_bd(ring); + /* Do not get FE for the packet or failed to alloc skb */ + if (unlikely(!ring->skb || err == -ENXIO)) { goto out; - - if (err == -ENXIO) { /* Do not get FE for the packet */ - goto out; - } else if (unlikely(err)) { /* Do jump the err */ - recv_bds += ring->pending_buf; - clean_count += ring->pending_buf; - ring->skb = NULL; - ring->pending_buf = 0; - continue; + } else if (likely(!err)) { + rx_fn(ring, ring->skb); + recv_pkts++; } - rx_fn(ring, skb); recv_bds += ring->pending_buf; - clean_count += ring->pending_buf; + unused_count += ring->pending_buf; ring->skb = NULL; ring->pending_buf = 0; - - recv_pkts++; } out: /* Make all data has been write before submit */ - if (clean_count + unused_count > 0) - hns3_nic_alloc_rx_buffers(ring, clean_count + unused_count); + if (unused_count > 0) + hns3_nic_alloc_rx_buffers(ring, unused_count); return recv_pkts; } @@ -3281,13 +3510,13 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) tqp_vector = &priv->tqp_vector[vector_i]; hns3_add_ring_to_group(&tqp_vector->tx_group, - priv->ring_data[i].ring); + &priv->ring[i]); hns3_add_ring_to_group(&tqp_vector->rx_group, - priv->ring_data[i + tqp_num].ring); + &priv->ring[i + tqp_num]); - priv->ring_data[i].ring->tqp_vector = tqp_vector; - priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector; + priv->ring[i].tqp_vector = tqp_vector; + priv->ring[i + tqp_num].tqp_vector = tqp_vector; tqp_vector->num_tqps++; } @@ -3393,26 +3622,25 @@ static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv) if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring) continue; - hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain); + /* Since the mapping can be overwritten, when fail to get the + * chain between vector and ring, we should go on to deal with + * the remaining options. + */ + if (hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain)) + dev_warn(priv->dev, "failed to get ring chain\n"); h->ae_algo->ops->unmap_ring_from_vector(h, tqp_vector->vector_irq, &vector_ring_chain); hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain); - if (tqp_vector->irq_init_flag == HNS3_VECTOR_INITED) { - irq_set_affinity_hint(tqp_vector->vector_irq, NULL); - free_irq(tqp_vector->vector_irq, tqp_vector); - tqp_vector->irq_init_flag = HNS3_VECTOR_NOT_INITED; - } - hns3_clear_ring_group(&tqp_vector->rx_group); hns3_clear_ring_group(&tqp_vector->tx_group); netif_napi_del(&priv->tqp_vector[i].napi); } } -static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv) +static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv) { struct hnae3_handle *h = priv->ae_handle; struct pci_dev *pdev = h->pdev; @@ -3424,35 +3652,28 @@ static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv) tqp_vector = &priv->tqp_vector[i]; ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq); if (ret) - return ret; + return; } devm_kfree(&pdev->dev, priv->tqp_vector); - return 0; } -static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, - unsigned int ring_type) +static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, + unsigned int ring_type) { - struct hns3_nic_ring_data *ring_data = priv->ring_data; int queue_num = priv->ae_handle->kinfo.num_tqps; - struct pci_dev *pdev = priv->ae_handle->pdev; struct hns3_enet_ring *ring; int desc_num; - ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL); - if (!ring) - return -ENOMEM; - if (ring_type == HNAE3_RING_TYPE_TX) { + ring = &priv->ring[q->tqp_index]; desc_num = priv->ae_handle->kinfo.num_tx_desc; - ring_data[q->tqp_index].ring = ring; - ring_data[q->tqp_index].queue_index = q->tqp_index; + ring->queue_index = q->tqp_index; ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET; } else { + ring = &priv->ring[q->tqp_index + queue_num]; desc_num = priv->ae_handle->kinfo.num_rx_desc; - ring_data[q->tqp_index + queue_num].ring = ring; - ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index; + ring->queue_index = q->tqp_index; ring->io_base = q->io_base; } @@ -3467,76 +3688,41 @@ static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, ring->desc_num = desc_num; ring->next_to_use = 0; ring->next_to_clean = 0; - - return 0; } -static int hns3_queue_to_ring(struct hnae3_queue *tqp, - struct hns3_nic_priv *priv) +static void hns3_queue_to_ring(struct hnae3_queue *tqp, + struct hns3_nic_priv *priv) { - int ret; - - ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX); - if (ret) - return ret; - - ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX); - if (ret) { - devm_kfree(priv->dev, priv->ring_data[tqp->tqp_index].ring); - return ret; - } - - return 0; + hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX); + hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX); } static int hns3_get_ring_config(struct hns3_nic_priv *priv) { struct hnae3_handle *h = priv->ae_handle; struct pci_dev *pdev = h->pdev; - int i, ret; + int i; - priv->ring_data = devm_kzalloc(&pdev->dev, - array3_size(h->kinfo.num_tqps, - sizeof(*priv->ring_data), - 2), - GFP_KERNEL); - if (!priv->ring_data) + priv->ring = devm_kzalloc(&pdev->dev, + array3_size(h->kinfo.num_tqps, + sizeof(*priv->ring), 2), + GFP_KERNEL); + if (!priv->ring) return -ENOMEM; - for (i = 0; i < h->kinfo.num_tqps; i++) { - ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv); - if (ret) - goto err; - } + for (i = 0; i < h->kinfo.num_tqps; i++) + hns3_queue_to_ring(h->kinfo.tqp[i], priv); return 0; -err: - while (i--) { - devm_kfree(priv->dev, priv->ring_data[i].ring); - devm_kfree(priv->dev, - priv->ring_data[i + h->kinfo.num_tqps].ring); - } - - devm_kfree(&pdev->dev, priv->ring_data); - priv->ring_data = NULL; - return ret; } static void hns3_put_ring_config(struct hns3_nic_priv *priv) { - struct hnae3_handle *h = priv->ae_handle; - int i; - - if (!priv->ring_data) + if (!priv->ring) return; - for (i = 0; i < h->kinfo.num_tqps; i++) { - devm_kfree(priv->dev, priv->ring_data[i].ring); - devm_kfree(priv->dev, - priv->ring_data[i + h->kinfo.num_tqps].ring); - } - devm_kfree(priv->dev, priv->ring_data); - priv->ring_data = NULL; + devm_kfree(priv->dev, priv->ring); + priv->ring = NULL; } static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring) @@ -3574,7 +3760,7 @@ out: return ret; } -static void hns3_fini_ring(struct hns3_enet_ring *ring) +void hns3_fini_ring(struct hns3_enet_ring *ring) { hns3_free_desc(ring); devm_kfree(ring_to_dev(ring), ring->desc_cb); @@ -3653,7 +3839,7 @@ static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv) for (j = 0; j < tc_info->tqp_count; j++) { struct hnae3_queue *q; - q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp; + q = priv->ring[tc_info->tqp_offset + j].tqp; hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG, tc_info->tc); } @@ -3668,21 +3854,21 @@ int hns3_init_all_ring(struct hns3_nic_priv *priv) int ret; for (i = 0; i < ring_num; i++) { - ret = hns3_alloc_ring_memory(priv->ring_data[i].ring); + ret = hns3_alloc_ring_memory(&priv->ring[i]); if (ret) { dev_err(priv->dev, "Alloc ring memory fail! ret=%d\n", ret); goto out_when_alloc_ring_memory; } - u64_stats_init(&priv->ring_data[i].ring->syncp); + u64_stats_init(&priv->ring[i].syncp); } return 0; out_when_alloc_ring_memory: for (j = i - 1; j >= 0; j--) - hns3_fini_ring(priv->ring_data[j].ring); + hns3_fini_ring(&priv->ring[j]); return -ENOMEM; } @@ -3693,30 +3879,31 @@ int hns3_uninit_all_ring(struct hns3_nic_priv *priv) int i; for (i = 0; i < h->kinfo.num_tqps; i++) { - hns3_fini_ring(priv->ring_data[i].ring); - hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring); + hns3_fini_ring(&priv->ring[i]); + hns3_fini_ring(&priv->ring[i + h->kinfo.num_tqps]); } return 0; } /* Set mac addr if it is configured. or leave it to the AE driver */ -static int hns3_init_mac_addr(struct net_device *netdev, bool init) +static int hns3_init_mac_addr(struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); struct hnae3_handle *h = priv->ae_handle; u8 mac_addr_temp[ETH_ALEN]; int ret = 0; - if (h->ae_algo->ops->get_mac_addr && init) { + if (h->ae_algo->ops->get_mac_addr) h->ae_algo->ops->get_mac_addr(h, mac_addr_temp); - ether_addr_copy(netdev->dev_addr, mac_addr_temp); - } /* Check if the MAC address is valid, if not get a random one */ - if (!is_valid_ether_addr(netdev->dev_addr)) { + if (!is_valid_ether_addr(mac_addr_temp)) { eth_hw_addr_random(netdev); dev_warn(priv->dev, "using random MAC address %pM\n", netdev->dev_addr); + } else { + ether_addr_copy(netdev->dev_addr, mac_addr_temp); + ether_addr_copy(netdev->perm_addr, mac_addr_temp); } if (h->ae_algo->ops->set_mac_addr) @@ -3784,14 +3971,14 @@ static void hns3_info_show(struct hns3_nic_priv *priv) struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo; dev_info(priv->dev, "MAC address: %pM\n", priv->netdev->dev_addr); - dev_info(priv->dev, "Task queue pairs numbers: %d\n", kinfo->num_tqps); - dev_info(priv->dev, "RSS size: %d\n", kinfo->rss_size); - dev_info(priv->dev, "Allocated RSS size: %d\n", kinfo->req_rss_size); - dev_info(priv->dev, "RX buffer length: %d\n", kinfo->rx_buf_len); - dev_info(priv->dev, "Desc num per TX queue: %d\n", kinfo->num_tx_desc); - dev_info(priv->dev, "Desc num per RX queue: %d\n", kinfo->num_rx_desc); - dev_info(priv->dev, "Total number of enabled TCs: %d\n", kinfo->num_tc); - dev_info(priv->dev, "Max mtu size: %d\n", priv->netdev->max_mtu); + dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps); + dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size); + dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size); + dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len); + dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc); + dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc); + dev_info(priv->dev, "Total number of enabled TCs: %u\n", kinfo->num_tc); + dev_info(priv->dev, "Max mtu size: %u\n", priv->netdev->max_mtu); } static int hns3_client_init(struct hnae3_handle *handle) @@ -3820,7 +4007,7 @@ static int hns3_client_init(struct hnae3_handle *handle) handle->kinfo.netdev = netdev; handle->priv = (void *)priv; - hns3_init_mac_addr(netdev, true); + hns3_init_mac_addr(netdev); hns3_set_default_feature(netdev); @@ -3854,7 +4041,7 @@ static int hns3_client_init(struct hnae3_handle *handle) ret = hns3_init_all_ring(priv); if (ret) { ret = -ENOMEM; - goto out_init_ring_data; + goto out_init_ring; } ret = hns3_init_phy(netdev); @@ -3867,6 +4054,18 @@ static int hns3_client_init(struct hnae3_handle *handle) goto out_reg_netdev_fail; } + /* the device can work without cpu rmap, only aRFS needs it */ + ret = hns3_set_rx_cpu_rmap(netdev); + if (ret) + dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret); + + ret = hns3_nic_init_irq(priv); + if (ret) { + dev_err(priv->dev, "init irq failed! ret=%d\n", ret); + hns3_free_rx_cpu_rmap(netdev); + goto out_init_irq_fail; + } + ret = hns3_client_start(handle); if (ret) { dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret); @@ -3888,17 +4087,20 @@ static int hns3_client_init(struct hnae3_handle *handle) return ret; out_client_start: + hns3_free_rx_cpu_rmap(netdev); + hns3_nic_uninit_irq(priv); +out_init_irq_fail: unregister_netdev(netdev); out_reg_netdev_fail: hns3_uninit_phy(netdev); out_init_phy: hns3_uninit_all_ring(priv); -out_init_ring_data: +out_init_ring: hns3_nic_uninit_vector_data(priv); out_init_vector_data: hns3_nic_dealloc_vector_data(priv); out_alloc_vector_data: - priv->ring_data = NULL; + priv->ring = NULL; out_get_ring_cfg: priv->ae_handle = NULL; free_netdev(netdev); @@ -3925,15 +4127,17 @@ static void hns3_client_uninit(struct hnae3_handle *handle, bool reset) goto out_netdev_free; } + hns3_free_rx_cpu_rmap(netdev); + + hns3_nic_uninit_irq(priv); + hns3_del_all_fd_rules(netdev, true); hns3_clear_all_ring(handle, true); hns3_nic_uninit_vector_data(priv); - ret = hns3_nic_dealloc_vector_data(priv); - if (ret) - netdev_err(netdev, "dealloc vector error\n"); + hns3_nic_dealloc_vector_data(priv); ret = hns3_uninit_all_ring(priv); if (ret) @@ -4059,7 +4263,7 @@ static int hns3_clear_rx_ring(struct hns3_enet_ring *ring) /* if alloc new buffer fail, exit directly * and reclear in up flow. */ - netdev_warn(ring->tqp->handle->kinfo.netdev, + netdev_warn(ring_to_netdev(ring), "reserve buffer map failed, ret = %d\n", ret); return ret; @@ -4105,10 +4309,10 @@ static void hns3_clear_all_ring(struct hnae3_handle *h, bool force) for (i = 0; i < h->kinfo.num_tqps; i++) { struct hns3_enet_ring *ring; - ring = priv->ring_data[i].ring; + ring = &priv->ring[i]; hns3_clear_tx_ring(ring); - ring = priv->ring_data[i + h->kinfo.num_tqps].ring; + ring = &priv->ring[i + h->kinfo.num_tqps]; /* Continue to clear other rings even if clearing some * rings failed. */ @@ -4132,16 +4336,16 @@ int hns3_nic_reset_all_ring(struct hnae3_handle *h) if (ret) return ret; - hns3_init_ring_hw(priv->ring_data[i].ring); + hns3_init_ring_hw(&priv->ring[i]); /* We need to clear tx ring here because self test will * use the ring and will not run down before up */ - hns3_clear_tx_ring(priv->ring_data[i].ring); - priv->ring_data[i].ring->next_to_clean = 0; - priv->ring_data[i].ring->next_to_use = 0; + hns3_clear_tx_ring(&priv->ring[i]); + priv->ring[i].next_to_clean = 0; + priv->ring[i].next_to_use = 0; - rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring; + rx_ring = &priv->ring[i + h->kinfo.num_tqps]; hns3_init_ring_hw(rx_ring); ret = hns3_clear_rx_ring(rx_ring); if (ret) @@ -4165,8 +4369,8 @@ int hns3_nic_reset_all_ring(struct hnae3_handle *h) static void hns3_store_coal(struct hns3_nic_priv *priv) { /* ethtool only support setting and querying one coal - * configuation for now, so save the vector 0' coal - * configuation here in order to restore it. + * configuration for now, so save the vector 0' coal + * configuration here in order to restore it. */ memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal, sizeof(struct hns3_enet_coalesce)); @@ -4260,17 +4464,32 @@ static int hns3_reset_notify_init_enet(struct hnae3_handle *handle) if (ret) goto err_uninit_vector; + /* the device can work without cpu rmap, only aRFS needs it */ + ret = hns3_set_rx_cpu_rmap(netdev); + if (ret) + dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret); + + ret = hns3_nic_init_irq(priv); + if (ret) { + dev_err(priv->dev, "init irq failed! ret=%d\n", ret); + hns3_free_rx_cpu_rmap(netdev); + goto err_init_irq_fail; + } + ret = hns3_client_start(handle); if (ret) { dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret); - goto err_uninit_ring; + goto err_client_start_fail; } set_bit(HNS3_NIC_STATE_INITED, &priv->state); return ret; -err_uninit_ring: +err_client_start_fail: + hns3_free_rx_cpu_rmap(netdev); + hns3_nic_uninit_irq(priv); +err_init_irq_fail: hns3_uninit_all_ring(priv); err_uninit_vector: hns3_nic_uninit_vector_data(priv); @@ -4288,7 +4507,7 @@ static int hns3_reset_notify_restore_enet(struct hnae3_handle *handle) bool vlan_filter_enable; int ret; - ret = hns3_init_mac_addr(netdev, false); + ret = hns3_init_mac_addr(netdev); if (ret) return ret; @@ -4320,6 +4539,8 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) return 0; } + hns3_free_rx_cpu_rmap(netdev); + hns3_nic_uninit_irq(priv); hns3_clear_all_ring(handle, true); hns3_reset_tx_queue(priv->ae_handle); @@ -4327,9 +4548,7 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) hns3_store_coal(priv); - ret = hns3_nic_dealloc_vector_data(priv); - if (ret) - netdev_err(netdev, "dealloc vector error\n"); + hns3_nic_dealloc_vector_data(priv); ret = hns3_uninit_all_ring(priv); if (ret) @@ -4368,6 +4587,30 @@ static int hns3_reset_notify(struct hnae3_handle *handle, return ret; } +static int hns3_change_channels(struct hnae3_handle *handle, u32 new_tqp_num, + bool rxfh_configured) +{ + int ret; + + ret = handle->ae_algo->ops->set_channels(handle, new_tqp_num, + rxfh_configured); + if (ret) { + dev_err(&handle->pdev->dev, + "Change tqp num(%u) fail.\n", new_tqp_num); + return ret; + } + + ret = hns3_reset_notify(handle, HNAE3_INIT_CLIENT); + if (ret) + return ret; + + ret = hns3_reset_notify(handle, HNAE3_UP_CLIENT); + if (ret) + hns3_reset_notify(handle, HNAE3_UNINIT_CLIENT); + + return ret; +} + int hns3_set_channels(struct net_device *netdev, struct ethtool_channels *ch) { @@ -4378,13 +4621,16 @@ int hns3_set_channels(struct net_device *netdev, u16 org_tqp_num; int ret; + if (hns3_nic_resetting(netdev)) + return -EBUSY; + if (ch->rx_count || ch->tx_count) return -EINVAL; if (new_tqp_num > hns3_get_max_available_channels(h) || new_tqp_num < 1) { dev_err(&netdev->dev, - "Change tqps fail, the tqp range is from 1 to %d", + "Change tqps fail, the tqp range is from 1 to %u", hns3_get_max_available_channels(h)); return -EINVAL; } @@ -4392,6 +4638,10 @@ int hns3_set_channels(struct net_device *netdev, if (kinfo->rss_size == new_tqp_num) return 0; + netif_dbg(h, drv, netdev, + "set channels: tqp_num=%u, rxfh=%d\n", + new_tqp_num, rxfh_configured); + ret = hns3_reset_notify(h, HNAE3_DOWN_CLIENT); if (ret) return ret; @@ -4401,24 +4651,46 @@ int hns3_set_channels(struct net_device *netdev, return ret; org_tqp_num = h->kinfo.num_tqps; - ret = h->ae_algo->ops->set_channels(h, new_tqp_num, rxfh_configured); + ret = hns3_change_channels(h, new_tqp_num, rxfh_configured); if (ret) { - ret = h->ae_algo->ops->set_channels(h, org_tqp_num, - rxfh_configured); - if (ret) { - /* If revert to old tqp failed, fatal error occurred */ - dev_err(&netdev->dev, - "Revert to old tqp num fail, ret=%d", ret); - return ret; + int ret1; + + netdev_warn(netdev, + "Change channels fail, revert to old value\n"); + ret1 = hns3_change_channels(h, org_tqp_num, rxfh_configured); + if (ret1) { + netdev_err(netdev, + "revert to old channel fail\n"); + return ret1; } - dev_info(&netdev->dev, - "Change tqp num fail, Revert to old tqp num"); - } - ret = hns3_reset_notify(h, HNAE3_INIT_CLIENT); - if (ret) + return ret; + } - return hns3_reset_notify(h, HNAE3_UP_CLIENT); + return 0; +} + +static const struct hns3_hw_error_info hns3_hw_err[] = { + { .type = HNAE3_PPU_POISON_ERROR, + .msg = "PPU poison" }, + { .type = HNAE3_CMDQ_ECC_ERROR, + .msg = "IMP CMDQ error" }, + { .type = HNAE3_IMP_RD_POISON_ERROR, + .msg = "IMP RD poison" }, +}; + +static void hns3_process_hw_error(struct hnae3_handle *handle, + enum hnae3_hw_error_type type) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(hns3_hw_err); i++) { + if (hns3_hw_err[i].type == type) { + dev_err(&handle->pdev->dev, "Detected %s!\n", + hns3_hw_err[i].msg); + break; + } + } } static const struct hnae3_client_ops client_ops = { @@ -4427,6 +4699,7 @@ static const struct hnae3_client_ops client_ops = { .link_status_change = hns3_link_status_change, .setup_tc = hns3_client_setup_tc, .reset_notify = hns3_reset_notify, + .process_hw_error = hns3_process_hw_error, }; /* hns3_init_module - Driver registration routine @@ -4441,7 +4714,7 @@ static int __init hns3_init_module(void) pr_info("%s: %s\n", hns3_driver_name, hns3_copyright); client.type = HNAE3_CLIENT_KNIC; - snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s", + snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH, "%s", hns3_driver_name); client.ops = &client_ops; |
