diff options
Diffstat (limited to 'drivers/infiniband')
26 files changed, 871 insertions, 263 deletions
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 9183d148d644..cb1d2ab13c66 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -207,6 +207,22 @@ int rdma_addr_size(struct sockaddr *addr) } EXPORT_SYMBOL(rdma_addr_size); +int rdma_addr_size_in6(struct sockaddr_in6 *addr) +{ + int ret = rdma_addr_size((struct sockaddr *) addr); + + return ret <= sizeof(*addr) ? ret : 0; +} +EXPORT_SYMBOL(rdma_addr_size_in6); + +int rdma_addr_size_kss(struct __kernel_sockaddr_storage *addr) +{ + int ret = rdma_addr_size((struct sockaddr *) addr); + + return ret <= sizeof(*addr) ? ret : 0; +} +EXPORT_SYMBOL(rdma_addr_size_kss); + static struct rdma_addr_client self; void rdma_addr_register_client(struct rdma_addr_client *client) @@ -586,6 +602,15 @@ static void process_one_req(struct work_struct *_work) list_del(&req->list); mutex_unlock(&lock); + /* + * Although the work will normally have been canceled by the + * workqueue, it can still be requeued as long as it is on the + * req_list, so it could have been requeued before we grabbed &lock. + * We need to cancel it after it is removed from req_list to really be + * sure it is safe to free. + */ + cancel_delayed_work(&req->work); + req->callback(req->status, (struct sockaddr *)&req->src_addr, req->addr, req->context); put_client(req->client); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index e66963ca58bd..6ab1059fed66 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -1334,7 +1334,7 @@ static bool validate_ipv6_net_dev(struct net_device *net_dev, IPV6_ADDR_LINKLOCAL; struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr, &src_addr->sin6_addr, net_dev->ifindex, - strict); + NULL, strict); bool ret; if (!rt) @@ -3069,7 +3069,8 @@ static int cma_port_is_unique(struct rdma_bind_list *bind_list, continue; /* different dest port -> unique */ - if (!cma_any_port(cur_daddr) && + if (!cma_any_port(daddr) && + !cma_any_port(cur_daddr) && (dport != cur_dport)) continue; @@ -3080,7 +3081,8 @@ static int cma_port_is_unique(struct rdma_bind_list *bind_list, continue; /* different dst address -> unique */ - if (!cma_any_addr(cur_daddr) && + if (!cma_any_addr(daddr) && + !cma_any_addr(cur_daddr) && cma_addr_cmp(daddr, cur_daddr)) continue; @@ -3378,13 +3380,13 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) } #endif } + daddr = cma_dst_addr(id_priv); + daddr->sa_family = addr->sa_family; + ret = cma_get_port(id_priv); if (ret) goto err2; - daddr = cma_dst_addr(id_priv); - daddr->sa_family = addr->sa_family; - return 0; err2: if (id_priv->cma_dev) @@ -4173,6 +4175,9 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, struct cma_multicast *mc; int ret; + if (!id->device) + return -EINVAL; + id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) && !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED)) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index bb065c9449be..b7459cf524e4 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -290,6 +290,7 @@ void ib_dealloc_device(struct ib_device *device) { WARN_ON(device->reg_state != IB_DEV_UNREGISTERED && device->reg_state != IB_DEV_UNINITIALIZED); + rdma_restrack_clean(&device->res); put_device(&device->dev); } EXPORT_SYMBOL(ib_dealloc_device); @@ -600,8 +601,6 @@ void ib_unregister_device(struct ib_device *device) } up_read(&lists_rwsem); - rdma_restrack_clean(&device->res); - ib_device_unregister_rdmacg(device); ib_device_unregister_sysfs(device); diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c index 5a52ec77940a..cc2966380c0c 100644 --- a/drivers/infiniband/core/roce_gid_mgmt.c +++ b/drivers/infiniband/core/roce_gid_mgmt.c @@ -403,10 +403,12 @@ static void enum_all_gids_of_dev_cb(struct ib_device *ib_dev, * our feet */ rtnl_lock(); + down_read(&net_rwsem); for_each_net(net) for_each_netdev(net, ndev) if (is_eth_port_of_netdev(ib_dev, port, rdma_ndev, ndev)) add_netdev_ips(ib_dev, port, rdma_ndev, ndev); + up_read(&net_rwsem); rtnl_unlock(); } diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c index 3a9d0f5b5881..d933336d7e01 100644 --- a/drivers/infiniband/core/ucma.c +++ b/drivers/infiniband/core/ucma.c @@ -132,7 +132,7 @@ static inline struct ucma_context *_ucma_find_context(int id, ctx = idr_find(&ctx_idr, id); if (!ctx) ctx = ERR_PTR(-ENOENT); - else if (ctx->file != file) + else if (ctx->file != file || !ctx->cm_id) ctx = ERR_PTR(-EINVAL); return ctx; } @@ -456,6 +456,7 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf, struct rdma_ucm_create_id cmd; struct rdma_ucm_create_id_resp resp; struct ucma_context *ctx; + struct rdma_cm_id *cm_id; enum ib_qp_type qp_type; int ret; @@ -476,10 +477,10 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf, return -ENOMEM; ctx->uid = cmd.uid; - ctx->cm_id = rdma_create_id(current->nsproxy->net_ns, - ucma_event_handler, ctx, cmd.ps, qp_type); - if (IS_ERR(ctx->cm_id)) { - ret = PTR_ERR(ctx->cm_id); + cm_id = rdma_create_id(current->nsproxy->net_ns, + ucma_event_handler, ctx, cmd.ps, qp_type); + if (IS_ERR(cm_id)) { + ret = PTR_ERR(cm_id); goto err1; } @@ -489,14 +490,19 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf, ret = -EFAULT; goto err2; } + + ctx->cm_id = cm_id; return 0; err2: - rdma_destroy_id(ctx->cm_id); + rdma_destroy_id(cm_id); err1: mutex_lock(&mut); idr_remove(&ctx_idr, ctx->id); mutex_unlock(&mut); + mutex_lock(&file->mut); + list_del(&ctx->list); + mutex_unlock(&file->mut); kfree(ctx); return ret; } @@ -626,6 +632,9 @@ static ssize_t ucma_bind_ip(struct ucma_file *file, const char __user *inbuf, if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; + if (!rdma_addr_size_in6(&cmd.addr)) + return -EINVAL; + ctx = ucma_get_ctx(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -639,22 +648,21 @@ static ssize_t ucma_bind(struct ucma_file *file, const char __user *inbuf, int in_len, int out_len) { struct rdma_ucm_bind cmd; - struct sockaddr *addr; struct ucma_context *ctx; int ret; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; - addr = (struct sockaddr *) &cmd.addr; - if (cmd.reserved || !cmd.addr_size || (cmd.addr_size != rdma_addr_size(addr))) + if (cmd.reserved || !cmd.addr_size || + cmd.addr_size != rdma_addr_size_kss(&cmd.addr)) return -EINVAL; ctx = ucma_get_ctx(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); - ret = rdma_bind_addr(ctx->cm_id, addr); + ret = rdma_bind_addr(ctx->cm_id, (struct sockaddr *) &cmd.addr); ucma_put_ctx(ctx); return ret; } @@ -670,13 +678,16 @@ static ssize_t ucma_resolve_ip(struct ucma_file *file, if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; + if (!rdma_addr_size_in6(&cmd.src_addr) || + !rdma_addr_size_in6(&cmd.dst_addr)) + return -EINVAL; + ctx = ucma_get_ctx(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr, - (struct sockaddr *) &cmd.dst_addr, - cmd.timeout_ms); + (struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms); ucma_put_ctx(ctx); return ret; } @@ -686,24 +697,23 @@ static ssize_t ucma_resolve_addr(struct ucma_file *file, int in_len, int out_len) { struct rdma_ucm_resolve_addr cmd; - struct sockaddr *src, *dst; struct ucma_context *ctx; int ret; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; - src = (struct sockaddr *) &cmd.src_addr; - dst = (struct sockaddr *) &cmd.dst_addr; - if (cmd.reserved || (cmd.src_size && (cmd.src_size != rdma_addr_size(src))) || - !cmd.dst_size || (cmd.dst_size != rdma_addr_size(dst))) + if (cmd.reserved || + (cmd.src_size && (cmd.src_size != rdma_addr_size_kss(&cmd.src_addr))) || + !cmd.dst_size || (cmd.dst_size != rdma_addr_size_kss(&cmd.dst_addr))) return -EINVAL; ctx = ucma_get_ctx(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); - ret = rdma_resolve_addr(ctx->cm_id, src, dst, cmd.timeout_ms); + ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr, + (struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms); ucma_put_ctx(ctx); return ret; } @@ -1156,6 +1166,11 @@ static ssize_t ucma_init_qp_attr(struct ucma_file *file, if (IS_ERR(ctx)) return PTR_ERR(ctx); + if (!ctx->cm_id->device) { + ret = -EINVAL; + goto out; + } + resp.qp_attr_mask = 0; memset(&qp_attr, 0, sizeof qp_attr); qp_attr.qp_state = cmd.qp_state; @@ -1297,7 +1312,7 @@ static ssize_t ucma_set_option(struct ucma_file *file, const char __user *inbuf, if (IS_ERR(ctx)) return PTR_ERR(ctx); - if (unlikely(cmd.optval > KMALLOC_MAX_SIZE)) + if (unlikely(cmd.optlen > KMALLOC_MAX_SIZE)) return -EINVAL; optval = memdup_user((void __user *) (unsigned long) cmd.optval, @@ -1321,7 +1336,7 @@ static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf, { struct rdma_ucm_notify cmd; struct ucma_context *ctx; - int ret; + int ret = -EINVAL; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; @@ -1330,7 +1345,9 @@ static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf, if (IS_ERR(ctx)) return PTR_ERR(ctx); - ret = rdma_notify(ctx->cm_id, (enum ib_event_type) cmd.event); + if (ctx->cm_id->device) + ret = rdma_notify(ctx->cm_id, (enum ib_event_type)cmd.event); + ucma_put_ctx(ctx); return ret; } @@ -1349,7 +1366,7 @@ static ssize_t ucma_process_join(struct ucma_file *file, return -ENOSPC; addr = (struct sockaddr *) &cmd->addr; - if (!cmd->addr_size || (cmd->addr_size != rdma_addr_size(addr))) + if (cmd->addr_size != rdma_addr_size(addr)) return -EINVAL; if (cmd->join_flags == RDMA_MC_JOIN_FLAG_FULLMEMBER) @@ -1416,7 +1433,10 @@ static ssize_t ucma_join_ip_multicast(struct ucma_file *file, join_cmd.response = cmd.response; join_cmd.uid = cmd.uid; join_cmd.id = cmd.id; - join_cmd.addr_size = rdma_addr_size((struct sockaddr *) &cmd.addr); + join_cmd.addr_size = rdma_addr_size_in6(&cmd.addr); + if (!join_cmd.addr_size) + return -EINVAL; + join_cmd.join_flags = RDMA_MC_JOIN_FLAG_FULLMEMBER; memcpy(&join_cmd.addr, &cmd.addr, join_cmd.addr_size); @@ -1432,6 +1452,9 @@ static ssize_t ucma_join_multicast(struct ucma_file *file, if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; + if (!rdma_addr_size_kss(&cmd.addr)) + return -EINVAL; + return ucma_process_join(file, &cmd, out_len); } diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h index 3eb7a8387116..96f76896488d 100644 --- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h @@ -57,8 +57,8 @@ #define BNXT_RE_PAGE_SIZE_8M BIT(BNXT_RE_PAGE_SHIFT_8M) #define BNXT_RE_PAGE_SIZE_1G BIT(BNXT_RE_PAGE_SHIFT_1G) -#define BNXT_RE_MAX_MR_SIZE_LOW BIT(BNXT_RE_PAGE_SHIFT_1G) -#define BNXT_RE_MAX_MR_SIZE_HIGH BIT(39) +#define BNXT_RE_MAX_MR_SIZE_LOW BIT_ULL(BNXT_RE_PAGE_SHIFT_1G) +#define BNXT_RE_MAX_MR_SIZE_HIGH BIT_ULL(39) #define BNXT_RE_MAX_MR_SIZE BNXT_RE_MAX_MR_SIZE_HIGH #define BNXT_RE_MAX_QPC_COUNT (64 * 1024) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 0dd75f449872..8301d7e5fa8c 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -3598,7 +3598,7 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, int umem_pgs, page_shift, rc; if (length > BNXT_RE_MAX_MR_SIZE) { - dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%ld\n", + dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%lld\n", length, BNXT_RE_MAX_MR_SIZE); return ERR_PTR(-ENOMEM); } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 06b42c880fd4..3a78faba8d91 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -243,7 +243,7 @@ static void bnxt_qplib_service_nq(unsigned long data) u32 sw_cons, raw_cons; u16 type; int budget = nq->budget; - u64 q_handle; + uintptr_t q_handle; /* Service the NQ until empty */ raw_cons = hwq->cons; @@ -526,7 +526,7 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, /* Configure the request */ req.dpi = cpu_to_le32(srq->dpi->dpi); - req.srq_handle = cpu_to_le64(srq); + req.srq_handle = cpu_to_le64((uintptr_t)srq); req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements); pbl = &srq->hwq.pbl[PBL_LVL_0]; diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index 7a9d0de89d6a..e96771ddc9a7 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -1217,6 +1217,7 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) if (ctx->dev) c4iw_remove(ctx); break; + case CXGB4_STATE_FATAL_ERROR: case CXGB4_STATE_START_RECOVERY: pr_info("%s: Fatal Error\n", pci_name(ctx->lldi.pdev)); if (ctx->dev) { diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 41fafebe3b0d..da4aa1a95b11 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -1153,7 +1153,7 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len) cinfo.sdma_ring_size = fd->cq->nentries; cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size; - trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, cinfo); + trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, &cinfo); if (copy_to_user((void __user *)arg, &cinfo, len)) return -EFAULT; diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h index 4eb4cc798035..e00c8a7d559c 100644 --- a/drivers/infiniband/hw/hfi1/trace_ctxts.h +++ b/drivers/infiniband/hw/hfi1/trace_ctxts.h @@ -106,7 +106,7 @@ TRACE_EVENT(hfi1_uctxtdata, TRACE_EVENT(hfi1_ctxt_info, TP_PROTO(struct hfi1_devdata *dd, unsigned int ctxt, unsigned int subctxt, - struct hfi1_ctxt_info cinfo), + struct hfi1_ctxt_info *cinfo), TP_ARGS(dd, ctxt, subctxt, cinfo), TP_STRUCT__entry(DD_DEV_ENTRY(dd) __field(unsigned int, ctxt) @@ -120,11 +120,11 @@ TRACE_EVENT(hfi1_ctxt_info, TP_fast_assign(DD_DEV_ASSIGN(dd); __entry->ctxt = ctxt; __entry->subctxt = subctxt; - __entry->egrtids = cinfo.egrtids; - __entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt; - __entry->rcvhdrq_size = cinfo.rcvhdrq_entsize; - __entry->sdma_ring_size = cinfo.sdma_ring_size; - __entry->rcvegr_size = cinfo.rcvegr_size; + __entry->egrtids = cinfo->egrtids; + __entry->rcvhdrq_cnt = cinfo->rcvhdrq_cnt; + __entry->rcvhdrq_size = cinfo->rcvhdrq_entsize; + __entry->sdma_ring_size = cinfo->sdma_ring_size; + __entry->rcvegr_size = cinfo->rcvegr_size; ), TP_printk("[%s] ctxt %u:%u " CINFO_FMT, __get_str(dev), diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index db2ff352d75f..ec638778661c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -4383,7 +4383,7 @@ err_dma_alloc_buf: eq->l0_dma = 0; if (mhop_num == 1) - for (i -= i; i >= 0; i--) + for (i -= 1; i >= 0; i--) dma_free_coherent(dev, buf_chk_sz, eq->buf[i], eq->buf_dma[i]); else if (mhop_num == 2) { diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile index bc6299697dda..d42b922bede8 100644 --- a/drivers/infiniband/hw/mlx5/Makefile +++ b/drivers/infiniband/hw/mlx5/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_MLX5_INFINIBAND) += mlx5_ib.o mlx5_ib-y := main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o cong.o mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o +mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 15457c9569a7..77d257ec899b 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -64,14 +64,9 @@ static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, enum mlx5_event type) } } -static void *get_cqe_from_buf(struct mlx5_ib_cq_buf *buf, int n, int size) -{ - return mlx5_buf_offset(&buf->buf, n * size); -} - static void *get_cqe(struct mlx5_ib_cq *cq, int n) { - return get_cqe_from_buf(&cq->buf, n, cq->mcq.cqe_sz); + return mlx5_frag_buf_get_wqe(&cq->buf.fbc, n); } static u8 sw_ownership_bit(int n, int nent) @@ -272,14 +267,8 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe) { - __be32 *p = (__be32 *)cqe; - int i; - mlx5_ib_warn(dev, "dump error cqe\n"); - for (i = 0; i < sizeof(*cqe) / 16; i++, p += 4) - pr_info("%08x %08x %08x %08x\n", be32_to_cpu(p[0]), - be32_to_cpu(p[1]), be32_to_cpu(p[2]), - be32_to_cpu(p[3])); + mlx5_dump_err_cqe(dev->mdev, cqe); } static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, @@ -404,7 +393,7 @@ static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf) { - mlx5_buf_free(dev->mdev, &buf->buf); + mlx5_frag_buf_free(dev->mdev, &buf->fbc.frag_buf); } static void get_sig_err_item(struct mlx5_sig_err_cqe *cqe, @@ -725,12 +714,25 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) return ret; } -static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf, - int nent, int cqe_size) +static int alloc_cq_frag_buf(struct mlx5_ib_dev *dev, + struct mlx5_ib_cq_buf *buf, + int nent, + int cqe_size) { + struct mlx5_frag_buf_ctrl *c = &buf->fbc; + struct mlx5_frag_buf *frag_buf = &c->frag_buf; + u32 cqc_buff[MLX5_ST_SZ_DW(cqc)] = {0}; int err; - err = mlx5_buf_alloc(dev->mdev, nent * cqe_size, &buf->buf); + MLX5_SET(cqc, cqc_buff, log_cq_size, ilog2(cqe_size)); + MLX5_SET(cqc, cqc_buff, cqe_sz, (cqe_size == 128) ? 1 : 0); + + mlx5_core_init_cq_frag_buf(&buf->fbc, cqc_buff); + + err = mlx5_frag_buf_alloc_node(dev->mdev, + nent * cqe_size, + frag_buf, + dev->mdev->priv.numa_node); if (err) return err; @@ -863,14 +865,15 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context) ib_umem_release(cq->buf.umem); } -static void init_cq_buf(struct mlx5_ib_cq *cq, struct mlx5_ib_cq_buf *buf) +static void init_cq_frag_buf(struct mlx5_ib_cq *cq, + struct mlx5_ib_cq_buf *buf) { int i; void *cqe; struct mlx5_cqe64 *cqe64; for (i = 0; i < buf->nent; i++) { - cqe = get_cqe_from_buf(buf, i, buf->cqe_size); + cqe = get_cqe(cq, i); cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64; cqe64->op_own = MLX5_CQE_INVALID << 4; } @@ -892,14 +895,15 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, cq->mcq.arm_db = cq->db.db + 1; cq->mcq.cqe_sz = cqe_size; - err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size); + err = alloc_cq_frag_buf(dev, &cq->buf, entries, cqe_size); if (err) goto err_db; - init_cq_buf(cq, &cq->buf); + init_cq_frag_buf(cq, &cq->buf); *inlen = MLX5_ST_SZ_BYTES(create_cq_in) + - MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * cq->buf.buf.npages; + MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * + cq->buf.fbc.frag_buf.npages; *cqb = kvzalloc(*inlen, GFP_KERNEL); if (!*cqb) { err = -ENOMEM; @@ -907,11 +911,12 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, } pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas); - mlx5_fill_page_array(&cq->buf.buf, pas); + mlx5_fill_page_frag_array(&cq->buf.fbc.frag_buf, pas); cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context); MLX5_SET(cqc, cqc, log_page_size, - cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); + cq->buf.fbc.frag_buf.page_shift - + MLX5_ADAPTER_PAGE_SHIFT); *index = dev->mdev->priv.uar->index; @@ -1213,11 +1218,11 @@ static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, if (!cq->resize_buf) return -ENOMEM; - err = alloc_cq_buf(dev, cq->resize_buf, entries, cqe_size); + err = alloc_cq_frag_buf(dev, cq->resize_buf, entries, cqe_size); if (err) goto ex; - init_cq_buf(cq, cq->resize_buf); + init_cq_frag_buf(cq, cq->resize_buf); return 0; @@ -1262,9 +1267,8 @@ static int copy_resize_cqes(struct mlx5_ib_cq *cq) } while ((scqe64->op_own >> 4) != MLX5_CQE_RESIZE_CQ) { - dcqe = get_cqe_from_buf(cq->resize_buf, - (i + 1) & (cq->resize_buf->nent), - dsize); + dcqe = mlx5_frag_buf_get_wqe(&cq->resize_buf->fbc, + (i + 1) & cq->resize_buf->nent); dcqe64 = dsize == 64 ? dcqe : dcqe + 64; sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent); memcpy(dcqe, scqe, dsize); @@ -1330,8 +1334,11 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) cqe_size = 64; err = resize_kernel(dev, cq, entries, cqe_size); if (!err) { - npas = cq->resize_buf->buf.npages; - page_shift = cq->resize_buf->buf.page_shift; + struct mlx5_frag_buf_ctrl *c; + + c = &cq->resize_buf->fbc; + npas = c->frag_buf.npages; + page_shift = c->frag_buf.page_shift; } } @@ -1352,7 +1359,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift, pas, 0); else - mlx5_fill_page_array(&cq->resize_buf->buf, pas); + mlx5_fill_page_frag_array(&cq->resize_buf->fbc.frag_buf, + pas); MLX5_SET(modify_cq_in, in, modify_field_select_resize_field_select.resize_field_select.resize_field_select, diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c new file mode 100644 index 000000000000..0e04fdddf670 --- /dev/null +++ b/drivers/infiniband/hw/mlx5/ib_rep.c @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright (c) 2018 Mellanox Technologies. All rights reserved. + */ + +#include "ib_rep.h" + +static const struct mlx5_ib_profile rep_profile = { + STAGE_CREATE(MLX5_IB_STAGE_INIT, + mlx5_ib_stage_init_init, + mlx5_ib_stage_init_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB, + mlx5_ib_stage_rep_flow_db_init, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_CAPS, + mlx5_ib_stage_caps_init, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB, + mlx5_ib_stage_rep_non_default_cb, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_ROCE, + mlx5_ib_stage_rep_roce_init, + mlx5_ib_stage_rep_roce_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES, + mlx5_ib_stage_dev_res_init, + mlx5_ib_stage_dev_res_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_COUNTERS, + mlx5_ib_stage_counters_init, + mlx5_ib_stage_counters_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_BFREG, + mlx5_ib_stage_bfrag_init, + mlx5_ib_stage_bfrag_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR, + NULL, + mlx5_ib_stage_pre_ib_reg_umr_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_IB_REG, + mlx5_ib_stage_ib_reg_init, + mlx5_ib_stage_ib_reg_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR, + mlx5_ib_stage_post_ib_reg_umr_init, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR, + mlx5_ib_stage_class_attr_init, + NULL), +}; + +static int +mlx5_ib_nic_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) +{ + return 0; +} + +static void +mlx5_ib_nic_rep_unload(struct mlx5_eswitch_rep *rep) +{ + rep->rep_if[REP_IB].priv = NULL; +} + +static int +mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) +{ + struct mlx5_ib_dev *ibdev; + + ibdev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*ibdev)); + if (!ibdev) + return -ENOMEM; + + ibdev->rep = rep; + ibdev->mdev = dev; + ibdev->num_ports = max(MLX5_CAP_GEN(dev, num_ports), + MLX5_CAP_GEN(dev, num_vhca_ports)); + if (!__mlx5_ib_add(ibdev, &rep_profile)) + return -EINVAL; + + rep->rep_if[REP_IB].priv = ibdev; + + return 0; +} + +static void +mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep) +{ + struct mlx5_ib_dev *dev; + + if (!rep->rep_if[REP_IB].priv) + return; + + dev = mlx5_ib_rep_to_dev(rep); + __mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX); + rep->rep_if[REP_IB].priv = NULL; +} + +static void *mlx5_ib_vport_get_proto_dev(struct mlx5_eswitch_rep *rep) +{ + return mlx5_ib_rep_to_dev(rep); +} + +static void mlx5_ib_rep_register_vf_vports(struct mlx5_ib_dev *dev) +{ + struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; + int total_vfs = MLX5_TOTAL_VPORTS(dev->mdev); + int vport; + + for (vport = 1; vport < total_vfs; vport++) { + struct mlx5_eswitch_rep_if rep_if = {}; + + rep_if.load = mlx5_ib_vport_rep_load; + rep_if.unload = mlx5_ib_vport_rep_unload; + rep_if.get_proto_dev = mlx5_ib_vport_get_proto_dev; + mlx5_eswitch_register_vport_rep(esw, vport, &rep_if, REP_IB); + } +} + +static void mlx5_ib_rep_unregister_vf_vports(struct mlx5_ib_dev *dev) +{ + struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; + int total_vfs = MLX5_TOTAL_VPORTS(dev->mdev); + int vport; + + for (vport = 1; vport < total_vfs; vport++) + mlx5_eswitch_unregister_vport_rep(esw, vport, REP_IB); +} + +void mlx5_ib_register_vport_reps(struct mlx5_ib_dev *dev) +{ + struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; + struct mlx5_eswitch_rep_if rep_if = {}; + + rep_if.load = mlx5_ib_nic_rep_load; + rep_if.unload = mlx5_ib_nic_rep_unload; + rep_if.get_proto_dev = mlx5_ib_vport_get_proto_dev; + rep_if.priv = dev; + + mlx5_eswitch_register_vport_rep(esw, 0, &rep_if, REP_IB); + + mlx5_ib_rep_register_vf_vports(dev); +} + +void mlx5_ib_unregister_vport_reps(struct mlx5_ib_dev *dev) +{ + struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; + + mlx5_ib_rep_unregister_vf_vports(dev); /* VFs vports */ + mlx5_eswitch_unregister_vport_rep(esw, 0, REP_IB); /* UPLINK PF*/ +} + +u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw) +{ + return mlx5_eswitch_mode(esw); +} + +struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw, + int vport_index) +{ + return mlx5_eswitch_get_proto_dev(esw, vport_index, REP_IB); +} + +struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw, + int vport_index) +{ + return mlx5_eswitch_get_proto_dev(esw, vport_index, REP_ETH); +} + +struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw) +{ + return mlx5_eswitch_uplink_get_proto_dev(esw, REP_IB); +} + +struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw, int vport) +{ + return mlx5_eswitch_vport_rep(esw, vport); +} + +int create_flow_rule_vport_sq(struct mlx5_ib_dev *dev, + struct mlx5_ib_sq *sq) +{ + struct mlx5_flow_handle *flow_rule; + struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; + + if (!dev->rep) + return 0; + + flow_rule = + mlx5_eswitch_add_send_to_vport_rule(esw, + dev->rep->vport, + sq->base.mqp.qpn); + if (IS_ERR(flow_rule)) + return PTR_ERR(flow_rule); + sq->flow_rule = flow_rule; + + return 0; +} diff --git a/drivers/infiniband/hw/mlx5/ib_rep.h b/drivers/infiniband/hw/mlx5/ib_rep.h new file mode 100644 index 000000000000..046fd942fd46 --- /dev/null +++ b/drivers/infiniband/hw/mlx5/ib_rep.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright (c) 2018 Mellanox Technologies. All rights reserved. + */ + +#ifndef __MLX5_IB_REP_H__ +#define __MLX5_IB_REP_H__ + +#include <linux/mlx5/eswitch.h> +#include "mlx5_ib.h" + +#ifdef CONFIG_MLX5_ESWITCH +u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw); +struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw, + int vport_index); +struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw); +struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw, + int vport_index); +void mlx5_ib_register_vport_reps(struct mlx5_ib_dev *dev); +void mlx5_ib_unregister_vport_reps(struct mlx5_ib_dev *dev); +int create_flow_rule_vport_sq(struct mlx5_ib_dev *dev, + struct mlx5_ib_sq *sq); +struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw, + int vport_index); +#else /* CONFIG_MLX5_ESWITCH */ +static inline u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw) +{ + return SRIOV_NONE; +} + +static inline +struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw, + int vport_index) +{ + return NULL; +} + +static inline +struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw) +{ + return NULL; +} + +static inline +struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw, + int vport_index) +{ + return NULL; +} + +static inline void mlx5_ib_register_vport_reps(struct mlx5_ib_dev *dev) {} +static inline void mlx5_ib_unregister_vport_reps(struct mlx5_ib_dev *dev) {} +static inline int create_flow_rule_vport_sq(struct mlx5_ib_dev *dev, + struct mlx5_ib_sq *sq) +{ + return 0; +} + +static inline +struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw, + int vport_index) +{ + return NULL; +} +#endif + +static inline +struct mlx5_ib_dev *mlx5_ib_rep_to_dev(struct mlx5_eswitch_rep *rep) +{ + return (struct mlx5_ib_dev *)rep->rep_if[REP_IB].priv; +} +#endif /* __MLX5_IB_REP_H__ */ diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 033b6af90de9..071fd9a7b919 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -57,7 +57,9 @@ #include <linux/in.h> #include <linux/etherdevice.h> #include "mlx5_ib.h" +#include "ib_rep.h" #include "cmd.h" +#include <linux/mlx5/fs_helpers.h> #define DRIVER_NAME "mlx5_ib" #define DRIVER_VERSION "5.0-0" @@ -130,7 +132,7 @@ static int get_port_state(struct ib_device *ibdev, int ret; memset(&attr, 0, sizeof(attr)); - ret = mlx5_ib_query_port(ibdev, port_num, &attr); + ret = ibdev->query_port(ibdev, port_num, &attr); if (!ret) *state = attr.state; return ret; @@ -154,10 +156,19 @@ static int mlx5_netdev_event(struct notifier_block *this, case NETDEV_REGISTER: case NETDEV_UNREGISTER: write_lock(&roce->netdev_lock); - - if (ndev->dev.parent == &mdev->pdev->dev) - roce->netdev = (event == NETDEV_UNREGISTER) ? + if (ibdev->rep) { + struct mlx5_eswitch *esw = ibdev->mdev->priv.eswitch; + struct net_device *rep_ndev; + + rep_ndev = mlx5_ib_get_rep_netdev(esw, + ibdev->rep->vport); + if (rep_ndev == ndev) + roce->netdev = (event == NETDEV_UNREGISTER) ? NULL : ndev; + } else if (ndev->dev.parent == &ibdev->mdev->pdev->dev) { + roce->netdev = (event == NETDEV_UNREGISTER) ? + NULL : ndev; + } write_unlock(&roce->netdev_lock); break; @@ -1272,6 +1283,22 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port, return ret; } +static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props) +{ + int ret; + + /* Only link layer == ethernet is valid for representors */ + ret = mlx5_query_port_roce(ibdev, port, props); + if (ret || !props) + return ret; + + /* We don't support GIDS */ + props->gid_tbl_len = 0; + + return ret; +} + static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { @@ -2290,11 +2317,9 @@ static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) offsetof(typeof(filter), field) -\ sizeof(filter.field)) -#define IPV4_VERSION 4 -#define IPV6_VERSION 6 static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c, u32 *match_v, const union ib_flow_spec *ib_spec, - u32 *tag_id, bool *is_drop) + struct mlx5_flow_act *action) { void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, misc_parameters); @@ -2377,7 +2402,7 @@ static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c, MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_version, 0xf); MLX5_SET(fte_match_set_lyr_2_4, headers_v, - ip_version, IPV4_VERSION); + ip_version, MLX5_FS_IPV4_VERSION); } else { MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype, 0xffff); @@ -2416,7 +2441,7 @@ static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c, MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_version, 0xf); MLX5_SET(fte_match_set_lyr_2_4, headers_v, - ip_version, IPV6_VERSION); + ip_version, MLX5_FS_IPV6_VERSION); } else { MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype, 0xffff); @@ -2512,13 +2537,14 @@ static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c, if (ib_spec->flow_tag.tag_id >= BIT(24)) return -EINVAL; - *tag_id = ib_spec->flow_tag.tag_id; + action->flow_tag = ib_spec->flow_tag.tag_id; + action->has_flow_tag = true; break; case IB_FLOW_SPEC_ACTION_DROP: if (FIELDS_NOT_SUPPORTED(ib_spec->drop, LAST_DROP_FIELD)) return -EOPNOTSUPP; - *is_drop = true; + action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; break; default: return -EINVAL; @@ -2635,7 +2661,7 @@ static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) ibflow); struct mlx5_ib_flow_handler *iter, *tmp; - mutex_lock(&dev->flow_db.lock); + mutex_lock(&dev->flow_db->lock); list_for_each_entry_safe(iter, tmp, &handler->list, list) { mlx5_del_flow_rules(iter->rule); @@ -2646,7 +2672,7 @@ static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) mlx5_del_flow_rules(handler->rule); put_flow_table(dev, handler->prio, true); - mutex_unlock(&dev->flow_db.lock); + mutex_unlock(&dev->flow_db->lock); kfree(handler); @@ -2695,7 +2721,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, MLX5_FLOW_NAMESPACE_BYPASS); num_entries = MLX5_FS_MAX_ENTRIES; num_groups = MLX5_FS_MAX_TYPES; - prio = &dev->flow_db.prios[priority]; + prio = &dev->flow_db->prios[priority]; } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { ns = mlx5_get_flow_namespace(dev->mdev, @@ -2703,7 +2729,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, build_leftovers_ft_param(&priority, &num_entries, &num_groups); - prio = &dev->flow_db.prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; + prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { if (!MLX5_CAP_FLOWTABLE(dev->mdev, allow_sniffer_and_nic_rx_shared_tir)) @@ -2713,7 +2739,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, MLX5_FLOW_NAMESPACE_SNIFFER_RX : MLX5_FLOW_NAMESPACE_SNIFFER_TX); - prio = &dev->flow_db.sniffer[ft_type]; + prio = &dev->flow_db->sniffer[ft_type]; priority = 0; num_entries = 1; num_groups = 1; @@ -2771,13 +2797,11 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, { struct mlx5_flow_table *ft = ft_prio->flow_table; struct mlx5_ib_flow_handler *handler; - struct mlx5_flow_act flow_act = {0}; + struct mlx5_flow_act flow_act = {.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG}; struct mlx5_flow_spec *spec; struct mlx5_flow_destination *rule_dst = dst; const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); unsigned int spec_index; - u32 flow_tag = MLX5_FS_DEFAULT_FLOW_TAG; - bool is_drop = false; int err = 0; int dest_num = 1; @@ -2796,7 +2820,7 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { err = parse_flow_attr(dev->mdev, spec->match_criteria, spec->match_value, - ib_flow, &flow_tag, &is_drop); + ib_flow, &flow_act); if (err < 0) goto free; @@ -2806,9 +2830,20 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, if (!flow_is_multicast_only(flow_attr)) set_underlay_qp(dev, spec, underlay_qpn); + if (dev->rep) { + void *misc; + + misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, + misc_parameters); + MLX5_SET(fte_match_set_misc, misc, source_port, + dev->rep->vport); + misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + misc_parameters); + MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); + } + spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); - if (is_drop) { - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; + if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { rule_dst = NULL; dest_num = 0; } else { @@ -2816,15 +2851,14 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; } - if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG && + if (flow_act.has_flow_tag && (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", - flow_tag, flow_attr->type); + flow_act.flow_tag, flow_attr->type); err = -EINVAL; goto free; } - flow_act.flow_tag = flow_tag; handler->rule = mlx5_add_flow_rules(ft, spec, &flow_act, rule_dst, dest_num); @@ -3003,7 +3037,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, if (!dst) return ERR_PTR(-ENOMEM); - mutex_lock(&dev->flow_db.lock); + mutex_lock(&dev->flow_db->lock); ft_prio = get_flow_table(dev, flow_attr, MLX5_IB_FT_RX); if (IS_ERR(ft_prio)) { @@ -3052,7 +3086,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, goto destroy_ft; } - mutex_unlock(&dev->flow_db.lock); + mutex_unlock(&dev->flow_db->lock); kfree(dst); return &handler->ibflow; @@ -3062,7 +3096,7 @@ destroy_ft: if (ft_prio_tx) put_flow_table(dev, ft_prio_tx, false); unlock: - mutex_unlock(&dev->flow_db.lock); + mutex_unlock(&dev->flow_db->lock); kfree(dst); kfree(handler); return ERR_PTR(err); @@ -3448,9 +3482,12 @@ static void destroy_umrc_res(struct mlx5_ib_dev *dev) if (err) mlx5_ib_warn(dev, "mr cache cleanup failed\n"); - mlx5_ib_destroy_qp(dev->umrc.qp); - ib_free_cq(dev->umrc.cq); - ib_dealloc_pd(dev->umrc.pd); + if (dev->umrc.qp) + mlx5_ib_destroy_qp(dev->umrc.qp); + if (dev->umrc.cq) + ib_free_cq(dev->umrc.cq); + if (dev->umrc.pd) + ib_dealloc_pd(dev->umrc.pd); } enum { @@ -3552,12 +3589,15 @@ static int create_umr_res(struct mlx5_ib_dev *dev) error_4: mlx5_ib_destroy_qp(qp); + dev->umrc.qp = NULL; error_3: ib_free_cq(cq); + dev->umrc.cq = NULL; error_2: ib_dealloc_pd(pd); + dev->umrc.pd = NULL; error_0: kfree(attr); @@ -3769,6 +3809,25 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, return 0; } +static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + + immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; + + err = ib_query_port(ibdev, port_num, &attr); + if (err) + return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; + + return 0; +} + static void get_dev_fw_str(struct ib_device *ibdev, char *str) { struct mlx5_ib_dev *dev = @@ -3799,7 +3858,7 @@ static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev) goto err_destroy_vport_lag; } - dev->flow_db.lag_demux_ft = ft; + dev->flow_db->lag_demux_ft = ft; return 0; err_destroy_vport_lag: @@ -3811,9 +3870,9 @@ static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev) { struct mlx5_core_dev *mdev = dev->mdev; - if (dev->flow_db.lag_demux_ft) { - mlx5_destroy_flow_table(dev->flow_db.lag_demux_ft); - dev->flow_db.lag_demux_ft = NULL; + if (dev->flow_db->lag_demux_ft) { + mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft); + dev->flow_db->lag_demux_ft = NULL; mlx5_cmd_destroy_vport_lag(mdev); } @@ -3845,14 +3904,10 @@ static int mlx5_enable_eth(struct mlx5_ib_dev *dev, u8 port_num) { int err; - err = mlx5_add_netdev_notifier(dev, port_num); - if (err) - return err; - if (MLX5_CAP_GEN(dev->mdev, roce)) { err = mlx5_nic_vport_enable_roce(dev->mdev); if (err) - goto err_unregister_netdevice_notifier; + return err; } err = mlx5_eth_lag_init(dev); @@ -3865,8 +3920,6 @@ err_disable_roce: if (MLX5_CAP_GEN(dev->mdev, roce)) mlx5_nic_vport_disable_roce(dev->mdev); -err_unregister_netdevice_notifier: - mlx5_remove_netdev_notifier(dev, port_num); return err; } @@ -4500,7 +4553,7 @@ static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev) mlx5_nic_vport_disable_roce(dev->mdev); } -static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev) +void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev) { mlx5_ib_cleanup_multiport_master(dev); #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING @@ -4509,7 +4562,7 @@ static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev) kfree(dev->port); } -static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) +int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) { struct mlx5_core_dev *mdev = dev->mdev; const char *name; @@ -4531,8 +4584,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) goto err_free_port; if (!mlx5_core_mp_enabled(mdev)) { - int i; - for (i = 1; i <= dev->num_ports; i++) { err = get_port_caps(dev, i); if (err) @@ -4561,7 +4612,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) dev->mdev->priv.eq_table.num_comp_vectors; dev->ib_dev.dev.parent = &mdev->pdev->dev; - mutex_init(&dev->flow_db.lock); mutex_init(&dev->cap_mask_mutex); INIT_LIST_HEAD(&dev->qp_list); spin_lock_init(&dev->reset_flow_resource_lock); @@ -4582,7 +4632,38 @@ err_free_port: return -ENOMEM; } -static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) +static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev) +{ + dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); + + if (!dev->flow_db) + return -ENOMEM; + + mutex_init(&dev->flow_db->lock); + + return 0; +} + +int mlx5_ib_stage_rep_flow_db_init(struct mlx5_ib_dev *dev) +{ + struct mlx5_ib_dev *nic_dev; + + nic_dev = mlx5_ib_get_uplink_ibdev(dev->mdev->priv.eswitch); + + if (!nic_dev) + return -EINVAL; + + dev->flow_db = nic_dev->flow_db; + + return 0; +} + +static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev) +{ + kfree(dev->flow_db); +} + +int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) { struct mlx5_core_dev *mdev = dev->mdev; int err; @@ -4623,7 +4704,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) (1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ); dev->ib_dev.query_device = mlx5_ib_query_device; - dev->ib_dev.query_port = mlx5_ib_query_port; dev->ib_dev.get_link_layer = mlx5_ib_port_link_layer; dev->ib_dev.query_gid = mlx5_ib_query_gid; dev->ib_dev.add_gid = mlx5_ib_add_gid; @@ -4666,7 +4746,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) dev->ib_dev.alloc_mr = mlx5_ib_alloc_mr; dev->ib_dev.map_mr_sg = mlx5_ib_map_mr_sg; dev->ib_dev.check_mr_status = mlx5_ib_check_mr_status; - dev->ib_dev.get_port_immutable = mlx5_port_immutable; dev->ib_dev.get_dev_fw_str = get_dev_fw_str; dev->ib_dev.get_vector_affinity = mlx5_ib_get_vector_affinity; if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads)) @@ -4717,6 +4796,80 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) return 0; } +static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev) +{ + dev->ib_dev.get_port_immutable = mlx5_port_immutable; + dev->ib_dev.query_port = mlx5_ib_query_port; + + return 0; +} + +int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev) +{ + dev->ib_dev.get_port_immutable = mlx5_port_rep_immutable; + dev->ib_dev.query_port = mlx5_ib_rep_query_port; + + return 0; +} + +static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev, + u8 port_num) +{ + int i; + + for (i = 0; i < dev->num_ports; i++) { + dev->roce[i].dev = dev; + dev->roce[i].native_port_num = i + 1; + dev->roce[i].last_port_state = IB_PORT_DOWN; + } + + dev->ib_dev.get_netdev = mlx5_ib_get_netdev; + dev->ib_dev.create_wq = mlx5_ib_create_wq; + dev->ib_dev.modify_wq = mlx5_ib_modify_wq; + dev->ib_dev.destroy_wq = mlx5_ib_destroy_wq; + dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table; + dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table; + + dev->ib_dev.uverbs_ex_cmd_mask |= + (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) | + (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) | + (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) | + (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) | + (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL); + + return mlx5_add_netdev_notifier(dev, port_num); +} + +static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev) +{ + u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1; + + mlx5_remove_netdev_notifier(dev, port_num); +} + +int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev) +{ + struct mlx5_core_dev *mdev = dev->mdev; + enum rdma_link_layer ll; + int port_type_cap; + int err = 0; + u8 port_num; + + port_num = mlx5_core_native_port_num(dev->mdev) - 1; + port_type_cap = MLX5_CAP_GEN(mdev, port_type); + ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); + + if (ll == IB_LINK_LAYER_ETHERNET) + err = mlx5_ib_stage_common_roce_init(dev, port_num); + + return err; +} + +void mlx5_ib_stage_rep_roce_cleanup(struct mlx5_ib_dev *dev) +{ + mlx5_ib_stage_common_roce_cleanup(dev); +} + static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev) { struct mlx5_core_dev *mdev = dev->mdev; @@ -4724,37 +4877,26 @@ static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev) int port_type_cap; u8 port_num; int err; - int i; port_num = mlx5_core_native_port_num(dev->mdev) - 1; port_type_cap = MLX5_CAP_GEN(mdev, port_type); ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); if (ll == IB_LINK_LAYER_ETHERNET) { - for (i = 0; i < dev->num_ports; i++) { - dev->roce[i].dev = dev; - dev->roce[i].native_port_num = i + 1; - dev->roce[i].last_port_state = IB_PORT_DOWN; - } + err = mlx5_ib_stage_common_roce_init(dev, port_num); + if (err) + return err; - dev->ib_dev.get_netdev = mlx5_ib_get_netdev; - dev->ib_dev.create_wq = mlx5_ib_create_wq; - dev->ib_dev.modify_wq = mlx5_ib_modify_wq; - dev->ib_dev.destroy_wq = mlx5_ib_destroy_wq; - dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table; - dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table; - dev->ib_dev.uverbs_ex_cmd_mask |= - (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) | - (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) | - (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) | - (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) | - (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL); err = mlx5_enable_eth(dev, port_num); if (err) - return err; + goto cleanup; } return 0; +cleanup: + mlx5_ib_stage_common_roce_cleanup(dev); + + return err; } static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev) @@ -4770,16 +4912,16 @@ static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev) if (ll == IB_LINK_LAYER_ETHERNET) { mlx5_disable_eth(dev); - mlx5_remove_netdev_notifier(dev, port_num); + mlx5_ib_stage_common_roce_cleanup(dev); } } -static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev) +int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev) { return create_dev_resources(&dev->devr); } -static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev) +void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev) { destroy_dev_resources(&dev->devr); } @@ -4791,7 +4933,7 @@ static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev) return mlx5_ib_odp_init_one(dev); } -static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev) +int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev) { if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) { dev->ib_dev.get_hw_stats = mlx5_ib_get_hw_stats; @@ -4803,7 +4945,7 @@ static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev) return 0; } -static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev) +void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev) { if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) mlx5_ib_dealloc_counters(dev); @@ -4834,7 +4976,7 @@ static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev) mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar); } -static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev) +int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev) { int err; @@ -4849,30 +4991,30 @@ static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev) return err; } -static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev) +void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev) { mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg); mlx5_free_bfreg(dev->mdev, &dev->bfreg); } -static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev) +int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev) { return ib_register_device(&dev->ib_dev, NULL); } -static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev) +void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev) { - ib_unregister_device(&dev->ib_dev); + destroy_umrc_res(dev); } -static int mlx5_ib_stage_umr_res_init(struct mlx5_ib_dev *dev) +void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev) { - return create_umr_res(dev); + ib_unregister_device(&dev->ib_dev); } -static void mlx5_ib_stage_umr_res_cleanup(struct mlx5_ib_dev *dev) +int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev) { - destroy_umrc_res(dev); + return create_umr_res(dev); } static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev) @@ -4887,7 +5029,7 @@ static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev) cancel_delay_drop(dev); } -static int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev) +int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev) { int err; int i; @@ -4902,9 +5044,21 @@ static int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev) return 0; } -static void __mlx5_ib_remove(struct mlx5_ib_dev *dev, - const struct mlx5_ib_profile *profile, - int stage) +static int mlx5_ib_stage_rep_reg_init(struct mlx5_ib_dev *dev) +{ + mlx5_ib_register_vport_reps(dev); + + return 0; +} + +static void mlx5_ib_stage_rep_reg_cleanup(struct mlx5_ib_dev *dev) +{ + mlx5_ib_unregister_vport_reps(dev); +} + +void __mlx5_ib_remove(struct mlx5_ib_dev *dev, + const struct mlx5_ib_profile *profile, + int stage) { /* Number of stages to cleanup */ while (stage) { @@ -4918,23 +5072,14 @@ static void __mlx5_ib_remove(struct mlx5_ib_dev *dev, static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num); -static void *__mlx5_ib_add(struct mlx5_core_dev *mdev, - const struct mlx5_ib_profile *profile) +void *__mlx5_ib_add(struct mlx5_ib_dev *dev, + const struct mlx5_ib_profile *profile) { - struct mlx5_ib_dev *dev; int err; int i; printk_once(KERN_INFO "%s", mlx5_version); - dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev)); - if (!dev) - return NULL; - - dev->mdev = mdev; - dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports), - MLX5_CAP_GEN(mdev, num_vhca_ports)); - for (i = 0; i < MLX5_IB_STAGE_MAX; i++) { if (profile->stage[i].init) { err = profile->stage[i].init(dev); @@ -4958,9 +5103,15 @@ static const struct mlx5_ib_profile pf_profile = { STAGE_CREATE(MLX5_IB_STAGE_INIT, mlx5_ib_stage_init_init, mlx5_ib_stage_init_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB, + mlx5_ib_stage_flow_db_init, + mlx5_ib_stage_flow_db_cleanup), STAGE_CREATE(MLX5_IB_STAGE_CAPS, mlx5_ib_stage_caps_init, NULL), + STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB, + mlx5_ib_stage_non_default_cb, + NULL), STAGE_CREATE(MLX5_IB_STAGE_ROCE, mlx5_ib_stage_roce_init, mlx5_ib_stage_roce_cleanup), @@ -4982,12 +5133,15 @@ static const struct mlx5_ib_profile pf_profile = { STAGE_CREATE(MLX5_IB_STAGE_BFREG, mlx5_ib_stage_bfrag_init, mlx5_ib_stage_bfrag_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR, + NULL, + mlx5_ib_stage_pre_ib_reg_umr_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), - STAGE_CREATE(MLX5_IB_STAGE_UMR_RESOURCES, - mlx5_ib_stage_umr_res_init, - mlx5_ib_stage_umr_res_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR, + mlx5_ib_stage_post_ib_reg_umr_init, + NULL), STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP, mlx5_ib_stage_delay_drop_init, mlx5_ib_stage_delay_drop_cleanup), @@ -4996,6 +5150,51 @@ static const struct mlx5_ib_profile pf_profile = { NULL), }; +static const struct mlx5_ib_profile nic_rep_profile = { + STAGE_CREATE(MLX5_IB_STAGE_INIT, + mlx5_ib_stage_init_init, + mlx5_ib_stage_init_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB, + mlx5_ib_stage_flow_db_init, + mlx5_ib_stage_flow_db_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_CAPS, + mlx5_ib_stage_caps_init, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB, + mlx5_ib_stage_rep_non_default_cb, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_ROCE, + mlx5_ib_stage_rep_roce_init, + mlx5_ib_stage_rep_roce_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES, + mlx5_ib_stage_dev_res_init, + mlx5_ib_stage_dev_res_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_COUNTERS, + mlx5_ib_stage_counters_init, + mlx5_ib_stage_counters_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_UAR, + mlx5_ib_stage_uar_init, + mlx5_ib_stage_uar_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_BFREG, + mlx5_ib_stage_bfrag_init, + mlx5_ib_stage_bfrag_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR, + NULL, + mlx5_ib_stage_pre_ib_reg_umr_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_IB_REG, + mlx5_ib_stage_ib_reg_init, + mlx5_ib_stage_ib_reg_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR, + mlx5_ib_stage_post_ib_reg_umr_init, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR, + mlx5_ib_stage_class_attr_init, + NULL), + STAGE_CREATE(MLX5_IB_STAGE_REP_REG, + mlx5_ib_stage_rep_reg_init, + mlx5_ib_stage_rep_reg_cleanup), +}; + static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num) { struct mlx5_ib_multiport_info *mpi; @@ -5041,8 +5240,11 @@ static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num) static void *mlx5_ib_add(struct mlx5_core_dev *mdev) { enum rdma_link_layer ll; + struct mlx5_ib_dev *dev; int port_type_cap; + printk_once(KERN_INFO "%s", mlx5_version); + port_type_cap = MLX5_CAP_GEN(mdev, port_type); ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); @@ -5052,7 +5254,22 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev) return mlx5_ib_add_slave_port(mdev, port_num); } - return __mlx5_ib_add(mdev, &pf_profile); + dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev)); + if (!dev) + return NULL; + + dev->mdev = mdev; + dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports), + MLX5_CAP_GEN(mdev, num_vhca_ports)); + + if (MLX5_VPORT_MANAGER(mdev) && + mlx5_ib_eswitch_mode(mdev->priv.eswitch) == SRIOV_OFFLOADS) { + dev->rep = mlx5_ib_vport_rep(mdev->priv.eswitch, 0); + + return __mlx5_ib_add(dev, &nic_rep_profile); + } + + return __mlx5_ib_add(dev, &pf_profile); } static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 139385129973..c33bf1523d67 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -343,6 +343,7 @@ struct mlx5_ib_sq { struct mlx5_ib_wq *sq; struct mlx5_ib_ubuffer ubuffer; struct mlx5_db *doorbell; + struct mlx5_flow_handle *flow_rule; u32 tisn; u8 state; }; @@ -371,7 +372,7 @@ struct mlx5_ib_qp { struct mlx5_ib_rss_qp rss_qp; struct mlx5_ib_dct dct; }; - struct mlx5_buf buf; + struct mlx5_frag_buf buf; struct mlx5_db db; struct mlx5_ib_wq rq; @@ -413,7 +414,7 @@ struct mlx5_ib_qp { }; struct mlx5_ib_cq_buf { - struct mlx5_buf buf; + struct mlx5_frag_buf_ctrl fbc; struct ib_umem *umem; int cqe_size; int nent; @@ -495,7 +496,7 @@ struct mlx5_ib_wc { struct mlx5_ib_srq { struct ib_srq ibsrq; struct mlx5_core_srq msrq; - struct mlx5_buf buf; + struct mlx5_frag_buf buf; struct mlx5_db db; u64 *wrid; /* protect SRQ hanlding @@ -731,7 +732,9 @@ struct mlx5_ib_delay_drop { enum mlx5_ib_stages { MLX5_IB_STAGE_INIT, + MLX5_IB_STAGE_FLOW_DB, MLX5_IB_STAGE_CAPS, + MLX5_IB_STAGE_NON_DEFAULT_CB, MLX5_IB_STAGE_ROCE, MLX5_IB_STAGE_DEVICE_RESOURCES, MLX5_IB_STAGE_ODP, @@ -739,10 +742,12 @@ enum mlx5_ib_stages { MLX5_IB_STAGE_CONG_DEBUGFS, MLX5_IB_STAGE_UAR, MLX5_IB_STAGE_BFREG, + MLX5_IB_STAGE_PRE_IB_REG_UMR, MLX5_IB_STAGE_IB_REG, - MLX5_IB_STAGE_UMR_RESOURCES, + MLX5_IB_STAGE_POST_IB_REG_UMR, MLX5_IB_STAGE_DELAY_DROP, MLX5_IB_STAGE_CLASS_ATTR, + MLX5_IB_STAGE_REP_REG, MLX5_IB_STAGE_MAX, }; @@ -797,7 +802,7 @@ struct mlx5_ib_dev { struct srcu_struct mr_srcu; u32 null_mkey; #endif - struct mlx5_ib_flow_db flow_db; + struct mlx5_ib_flow_db *flow_db; /* protect resources needed as part of reset flow */ spinlock_t reset_flow_resource_lock; struct list_head qp_list; @@ -807,6 +812,7 @@ struct mlx5_ib_dev { struct mlx5_sq_bfreg fp_bfreg; struct mlx5_ib_delay_drop delay_drop; const struct mlx5_ib_profile *profile; + struct mlx5_eswitch_rep *rep; /* protect the user_td */ struct mutex lb_mutex; @@ -1049,6 +1055,31 @@ static inline void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* Needed for rep profile */ +int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev); +void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_rep_flow_db_init(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev); +void mlx5_ib_stage_rep_roce_cleanup(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev); +void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev); +void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev); +void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev); +void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev); +void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev); +int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev); +void __mlx5_ib_remove(struct mlx5_ib_dev *dev, + const struct mlx5_ib_profile *profile, + int stage); +void *__mlx5_ib_add(struct mlx5_ib_dev *dev, + const struct mlx5_ib_profile *profile); + int mlx5_ib_get_vf_config(struct ib_device *device, int vf, u8 port, struct ifla_vf_info *info); int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf, diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 1961c6a45437..654bc31bc428 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -587,7 +587,7 @@ static void clean_keys(struct mlx5_ib_dev *dev, int c) static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) { - if (!mlx5_debugfs_root) + if (!mlx5_debugfs_root || dev->rep) return; debugfs_remove_recursive(dev->cache.root); @@ -600,7 +600,7 @@ static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev) struct mlx5_cache_ent *ent; int i; - if (!mlx5_debugfs_root) + if (!mlx5_debugfs_root || dev->rep) return 0; cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root); @@ -690,6 +690,7 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) MLX5_IB_UMR_OCTOWORD; ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && + !dev->rep && mlx5_core_is_pf(dev->mdev)) ent->limit = dev->mdev->profile->mr_cache[i].limit; else @@ -739,6 +740,9 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) { int i; + if (!dev->cache.wq) + return 0; + dev->cache.stopped = 1; flush_workqueue(dev->cache.wq); @@ -838,7 +842,8 @@ static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length, *umem = ib_umem_get(pd->uobject->context, start, length, access_flags, 0); err = PTR_ERR_OR_ZERO(*umem); - if (err < 0) { + if (err) { + *umem = NULL; mlx5_ib_err(dev, "umem get failed (%d)\n", err); return err; } @@ -1415,6 +1420,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, if (err) { mlx5_ib_warn(dev, "Failed to rereg UMR\n"); ib_umem_release(mr->umem); + mr->umem = NULL; clean_mr(dev, mr); return err; } @@ -1498,14 +1504,11 @@ static int clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) u32 key = mr->mmkey.key; err = destroy_mkey(dev, mr); - kfree(mr); if (err) { mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n", key, err); return err; } - } else { - mlx5_mr_cache_free(dev, mr); } return 0; @@ -1548,6 +1551,11 @@ static int dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) atomic_sub(npages, &dev->mdev->priv.reg_pages); } + if (!mr->allocated_from_cache) + kfree(mr); + else + mlx5_mr_cache_free(dev, mr); + return 0; } diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 36197fbac63a..0d0b0b8dad98 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -36,6 +36,7 @@ #include <rdma/ib_user_verbs.h> #include <linux/mlx5/fs.h> #include "mlx5_ib.h" +#include "ib_rep.h" /* not supported currently */ static int wq_signature; @@ -1082,6 +1083,13 @@ static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev, mlx5_core_destroy_tis(dev->mdev, sq->tisn); } +static void destroy_flow_rule_vport_sq(struct mlx5_ib_dev *dev, + struct mlx5_ib_sq *sq) +{ + if (sq->flow_rule) + mlx5_del_flow_rules(sq->flow_rule); +} + static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, struct mlx5_ib_sq *sq, void *qpin, struct ib_pd *pd) @@ -1145,8 +1153,15 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, if (err) goto err_umem; + err = create_flow_rule_vport_sq(dev, sq); + if (err) + goto err_flow; + return 0; +err_flow: + mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp); + err_umem: ib_umem_release(sq->ubuffer.umem); sq->ubuffer.umem = NULL; @@ -1157,11 +1172,12 @@ err_umem: static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev, struct mlx5_ib_sq *sq) { + destroy_flow_rule_vport_sq(dev, sq); mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp); ib_umem_release(sq->ubuffer.umem); } -static int get_rq_pas_size(void *qpc) +static size_t get_rq_pas_size(void *qpc) { u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12; u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride); @@ -1177,7 +1193,8 @@ static int get_rq_pas_size(void *qpc) } static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev, - struct mlx5_ib_rq *rq, void *qpin) + struct mlx5_ib_rq *rq, void *qpin, + size_t qpinlen) { struct mlx5_ib_qp *mqp = rq->base.container_mibqp; __be64 *pas; @@ -1186,9 +1203,12 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev, void *rqc; void *wq; void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc); - int inlen; + size_t rq_pas_size = get_rq_pas_size(qpc); + size_t inlen; int err; - u32 rq_pas_size = get_rq_pas_size(qpc); + + if (qpinlen < rq_pas_size + MLX5_BYTE_OFF(create_qp_in, pas)) + return -EINVAL; inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size; in = kvzalloc(inlen, GFP_KERNEL); @@ -1263,6 +1283,10 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev, if (tunnel_offload_en) MLX5_SET(tirc, tirc, tunneled_offload_en, 1); + if (dev->rep) + MLX5_SET(tirc, tirc, self_lb_block, + MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_); + err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn); kvfree(in); @@ -1277,7 +1301,7 @@ static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev, } static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, - u32 *in, + u32 *in, size_t inlen, struct ib_pd *pd) { struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; @@ -1309,7 +1333,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING; if (qp->flags & MLX5_IB_QP_PCI_WRITE_END_PADDING) rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING; - err = create_raw_packet_qp_rq(dev, rq, in); + err = create_raw_packet_qp_rq(dev, rq, in, inlen); if (err) goto err_destroy_sq; @@ -1554,6 +1578,10 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields); create_tir: + if (dev->rep) + MLX5_SET(tirc, tirc, self_lb_block, + MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_); + err = mlx5_core_create_tir(dev->mdev, in, inlen, &qp->rss_qp.tirn); if (err) @@ -1872,11 +1900,16 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd, } } + if (inlen < 0) { + err = -EINVAL; + goto err; + } + if (init_attr->qp_type == IB_QPT_RAW_PACKET || qp->flags & MLX5_IB_QP_UNDERLAY) { qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr; raw_packet_qp_copy_info(qp, &qp->raw_packet_qp); - err = create_raw_packet_qp(dev, qp, in, pd); + err = create_raw_packet_qp(dev, qp, in, inlen, pd); } else { err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen); } @@ -2134,7 +2167,6 @@ static struct ib_qp *mlx5_ib_create_dct(struct ib_pd *pd, struct ib_qp_init_attr *attr, struct mlx5_ib_create_qp *ucmd) { - struct mlx5_ib_dev *dev; struct mlx5_ib_qp *qp; int err = 0; u32 uidx = MLX5_IB_DEFAULT_UIDX; @@ -2143,8 +2175,6 @@ static struct ib_qp *mlx5_ib_create_dct(struct ib_pd *pd, if (!attr->srq || !attr->recv_cq) return ERR_PTR(-EINVAL); - dev = to_mdev(pd->device); - err = get_qp_user_index(to_mucontext(pd->uobject->context), ucmd, sizeof(*ucmd), &uidx); if (err) @@ -4709,26 +4739,14 @@ static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev, struct mlx5_ib_sq *sq, u8 *sq_state) { - void *out; - void *sqc; - int inlen; int err; - inlen = MLX5_ST_SZ_BYTES(query_sq_out); - out = kvzalloc(inlen, GFP_KERNEL); - if (!out) - return -ENOMEM; - - err = mlx5_core_query_sq(dev->mdev, sq->base.mqp.qpn, out); + err = mlx5_core_query_sq_state(dev->mdev, sq->base.mqp.qpn, sq_state); if (err) goto out; - - sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context); - *sq_state = MLX5_GET(sqc, sqc, state); sq->state = *sq_state; out: - kvfree(out); return err; } diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index 6d5fadad9090..3c7522d025f2 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -241,8 +241,8 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, { struct mlx5_ib_dev *dev = to_mdev(pd->device); struct mlx5_ib_srq *srq; - int desc_size; - int buf_size; + size_t desc_size; + size_t buf_size; int err; struct mlx5_srq_attr in = {0}; __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); @@ -266,15 +266,18 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); + if (desc_size == 0 || srq->msrq.max_gs > desc_size) + return ERR_PTR(-EINVAL); desc_size = roundup_pow_of_two(desc_size); - desc_size = max_t(int, 32, desc_size); + desc_size = max_t(size_t, 32, desc_size); + if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) + return ERR_PTR(-EINVAL); srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / sizeof(struct mlx5_wqe_data_seg); srq->msrq.wqe_shift = ilog2(desc_size); buf_size = srq->msrq.max * desc_size; - mlx5_ib_dbg(dev, "desc_size 0x%x, req wr 0x%x, srq size 0x%x, max_gs 0x%x, max_avail_gather 0x%x\n", - desc_size, init_attr->attr.max_wr, srq->msrq.max, srq->msrq.max_gs, - srq->msrq.max_avail_gather); + if (buf_size < desc_size) + return ERR_PTR(-EINVAL); in.type = init_attr->srq_type; if (pd->uobject) diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index db4bf97c0e15..f9a645c869ce 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -90,8 +90,8 @@ static struct net_device *qedr_get_netdev(struct ib_device *dev, u8 port_num) dev_hold(qdev->ndev); /* The HW vendor's device driver must guarantee - * that this function returns NULL before the net device reaches - * NETDEV_UNREGISTER_FINAL state. + * that this function returns NULL before the net device has finished + * NETDEV_UNREGISTER state. */ return qdev->ndev; } @@ -833,7 +833,8 @@ static struct qedr_dev *qedr_add(struct qed_dev *cdev, struct pci_dev *pdev, dev->num_cnq = dev->ops->rdma_get_min_cnq_msix(cdev); if (!dev->num_cnq) { - DP_ERR(dev, "not enough CNQ resources.\n"); + DP_ERR(dev, "Failed. At least one CNQ is required.\n"); + rc = -ENOMEM; goto init_err; } diff --git a/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h b/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h index 78b49002fbd2..b816c80df50b 100644 --- a/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h +++ b/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h @@ -45,7 +45,7 @@ struct rdma_cqe_responder { __le32 imm_data_or_inv_r_Key; __le32 length; __le32 imm_data_hi; - __le16 rq_cons; + __le16 rq_cons_or_srq_id; u8 flags; #define RDMA_CQE_RESPONDER_TOGGLE_BIT_MASK 0x1 #define RDMA_CQE_RESPONDER_TOGGLE_BIT_SHIFT 0 @@ -115,6 +115,7 @@ enum rdma_cqe_requester_status_enum { RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR, RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR, RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR, + RDMA_CQE_REQ_STS_XRC_VOILATION_ERR, MAX_RDMA_CQE_REQUESTER_STATUS_ENUM }; @@ -136,6 +137,7 @@ enum rdma_cqe_type { RDMA_CQE_TYPE_REQUESTER, RDMA_CQE_TYPE_RESPONDER_RQ, RDMA_CQE_TYPE_RESPONDER_SRQ, + RDMA_CQE_TYPE_RESPONDER_XRC_SRQ, RDMA_CQE_TYPE_INVALID, MAX_RDMA_CQE_TYPE }; diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 875b17272d65..f9c3cc71f5c0 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -1841,14 +1841,15 @@ static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) static int qedr_update_qp_state(struct qedr_dev *dev, struct qedr_qp *qp, + enum qed_roce_qp_state cur_state, enum qed_roce_qp_state new_state) { int status = 0; - if (new_state == qp->state) + if (new_state == cur_state) return 0; - switch (qp->state) { + switch (cur_state) { case QED_ROCE_QP_STATE_RESET: switch (new_state) { case QED_ROCE_QP_STATE_INIT: @@ -1955,6 +1956,7 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); enum ib_qp_state old_qp_state, new_qp_state; + enum qed_roce_qp_state cur_state; int rc = 0; DP_DEBUG(dev, QEDR_MSG_QP, @@ -2086,18 +2088,23 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, SET_FIELD(qp_params.modify_flags, QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); - qp_params.ack_timeout = attr->timeout; - if (attr->timeout) { - u32 temp; - - temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; - /* FW requires [msec] */ - qp_params.ack_timeout = temp; - } else { - /* Infinite */ + /* The received timeout value is an exponent used like this: + * "12.7.34 LOCAL ACK TIMEOUT + * Value representing the transport (ACK) timeout for use by + * the remote, expressed as: 4.096 * 2^timeout [usec]" + * The FW expects timeout in msec so we need to divide the usec + * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, + * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). + * The value of zero means infinite so we use a 'max_t' to make + * sure that sub 1 msec values will be configured as 1 msec. + */ + if (attr->timeout) + qp_params.ack_timeout = + 1 << max_t(int, attr->timeout - 8, 0); + else qp_params.ack_timeout = 0; - } } + if (attr_mask & IB_QP_RETRY_CNT) { SET_FIELD(qp_params.modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); @@ -2170,13 +2177,25 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, qp->dest_qp_num = attr->dest_qp_num; } + cur_state = qp->state; + + /* Update the QP state before the actual ramrod to prevent a race with + * fast path. Modifying the QP state to error will cause the device to + * flush the CQEs and while polling the flushed CQEs will considered as + * a potential issue if the QP isn't in error state. + */ + if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && + !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) + qp->state = QED_ROCE_QP_STATE_ERR; + if (qp->qp_type != IB_QPT_GSI) rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, qp->qed_qp, &qp_params); if (attr_mask & IB_QP_STATE) { if ((qp->qp_type != IB_QPT_GSI) && (!udata)) - rc = qedr_update_qp_state(dev, qp, qp_params.new_state); + rc = qedr_update_qp_state(dev, qp, cur_state, + qp_params.new_state); qp->state = qp_params.new_state; } @@ -3695,7 +3714,7 @@ static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, struct rdma_cqe_responder *resp, int *update) { - if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { + if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { consume_cqe(cq); *update |= 1; } @@ -3710,7 +3729,7 @@ static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { cnt = process_resp_flush(qp, cq, num_entries, wc, - resp->rq_cons); + resp->rq_cons_or_srq_id); try_consume_resp_cqe(cq, qp, resp, update); } else { cnt = process_resp_one(dev, qp, cq, wc, resp); diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index f45e99a938e0..ca5638091b55 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -95,24 +95,6 @@ void usnic_ib_log_vf(struct usnic_ib_vf *vf) } /* Start of netdev section */ -static inline const char *usnic_ib_netdev_event_to_string(unsigned long event) -{ - const char *event2str[] = {"NETDEV_NONE", "NETDEV_UP", "NETDEV_DOWN", - "NETDEV_REBOOT", "NETDEV_CHANGE", - "NETDEV_REGISTER", "NETDEV_UNREGISTER", "NETDEV_CHANGEMTU", - "NETDEV_CHANGEADDR", "NETDEV_GOING_DOWN", "NETDEV_FEAT_CHANGE", - "NETDEV_BONDING_FAILOVER", "NETDEV_PRE_UP", - "NETDEV_PRE_TYPE_CHANGE", "NETDEV_POST_TYPE_CHANGE", - "NETDEV_POST_INT", "NETDEV_UNREGISTER_FINAL", "NETDEV_RELEASE", - "NETDEV_NOTIFY_PEERS", "NETDEV_JOIN" - }; - - if (event >= ARRAY_SIZE(event2str)) - return "UNKNOWN_NETDEV_EVENT"; - else - return event2str[event]; -} - static void usnic_ib_qp_grp_modify_active_to_err(struct usnic_ib_dev *us_ibdev) { struct usnic_ib_ucontext *ctx; @@ -185,7 +167,7 @@ static void usnic_ib_handle_usdev_event(struct usnic_ib_dev *us_ibdev, ib_dispatch_event(&ib_event); } else { usnic_dbg("Ignoring %s on %s\n", - usnic_ib_netdev_event_to_string(event), + netdev_cmd_to_name(event), us_ibdev->ib_dev.name); } break; @@ -222,7 +204,7 @@ static void usnic_ib_handle_usdev_event(struct usnic_ib_dev *us_ibdev, break; default: usnic_dbg("Ignoring event %s on %s", - usnic_ib_netdev_event_to_string(event), + netdev_cmd_to_name(event), us_ibdev->ib_dev.name); } mutex_unlock(&us_ibdev->usdev_lock); @@ -264,7 +246,7 @@ static int usnic_ib_handle_inet_event(struct usnic_ib_dev *us_ibdev, switch (event) { case NETDEV_DOWN: usnic_info("%s via ip notifiers", - usnic_ib_netdev_event_to_string(event)); + netdev_cmd_to_name(event)); usnic_fwd_del_ipaddr(us_ibdev->ufdev); usnic_ib_qp_grp_modify_active_to_err(us_ibdev); ib_event.event = IB_EVENT_GID_CHANGE; @@ -275,7 +257,7 @@ static int usnic_ib_handle_inet_event(struct usnic_ib_dev *us_ibdev, case NETDEV_UP: usnic_fwd_add_ipaddr(us_ibdev->ufdev, ifa->ifa_address); usnic_info("%s via ip notifiers: ip %pI4", - usnic_ib_netdev_event_to_string(event), + netdev_cmd_to_name(event), &us_ibdev->ufdev->inaddr); ib_event.event = IB_EVENT_GID_CHANGE; ib_event.device = &us_ibdev->ib_dev; @@ -284,7 +266,7 @@ static int usnic_ib_handle_inet_event(struct usnic_ib_dev *us_ibdev, break; default: usnic_info("Ignoring event %s on %s", - usnic_ib_netdev_event_to_string(event), + netdev_cmd_to_name(event), us_ibdev->ib_dev.name); } mutex_unlock(&us_ibdev->usdev_lock); diff --git a/drivers/infiniband/hw/usnic/usnic_transport.c b/drivers/infiniband/hw/usnic/usnic_transport.c index de318389a301..67de94343cb4 100644 --- a/drivers/infiniband/hw/usnic/usnic_transport.c +++ b/drivers/infiniband/hw/usnic/usnic_transport.c @@ -174,14 +174,13 @@ void usnic_transport_put_socket(struct socket *sock) int usnic_transport_sock_get_addr(struct socket *sock, int *proto, uint32_t *addr, uint16_t *port) { - int len; int err; struct sockaddr_in sock_addr; err = sock->ops->getname(sock, (struct sockaddr *)&sock_addr, - &len, 0); - if (err) + 0); + if (err < 0) return err; if (sock_addr.sin_family != AF_INET) |
