summaryrefslogtreecommitdiffstats
path: root/drivers/nvme/host/rdma.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/nvme/host/rdma.c')
-rw-r--r--drivers/nvme/host/rdma.c71
1 files changed, 45 insertions, 26 deletions
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index c2c2c28e6eb5..3d25add36d91 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -54,7 +54,6 @@
struct nvme_rdma_device {
struct ib_device *dev;
struct ib_pd *pd;
- struct ib_mr *mr;
struct kref ref;
struct list_head entry;
};
@@ -84,6 +83,7 @@ enum nvme_rdma_queue_flags {
NVME_RDMA_Q_CONNECTED = (1 << 0),
NVME_RDMA_IB_QUEUE_ALLOCATED = (1 << 1),
NVME_RDMA_Q_DELETING = (1 << 2),
+ NVME_RDMA_Q_LIVE = (1 << 3),
};
struct nvme_rdma_queue {
@@ -408,10 +408,7 @@ static void nvme_rdma_free_dev(struct kref *ref)
list_del(&ndev->entry);
mutex_unlock(&device_list_mutex);
- if (!register_always)
- ib_dereg_mr(ndev->mr);
ib_dealloc_pd(ndev->pd);
-
kfree(ndev);
}
@@ -444,24 +441,16 @@ nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
ndev->dev = cm_id->device;
kref_init(&ndev->ref);
- ndev->pd = ib_alloc_pd(ndev->dev);
+ ndev->pd = ib_alloc_pd(ndev->dev,
+ register_always ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
if (IS_ERR(ndev->pd))
goto out_free_dev;
- if (!register_always) {
- ndev->mr = ib_get_dma_mr(ndev->pd,
- IB_ACCESS_LOCAL_WRITE |
- IB_ACCESS_REMOTE_READ |
- IB_ACCESS_REMOTE_WRITE);
- if (IS_ERR(ndev->mr))
- goto out_free_pd;
- }
-
if (!(ndev->dev->attrs.device_cap_flags &
IB_DEVICE_MEM_MGT_EXTENSIONS)) {
dev_err(&ndev->dev->dev,
"Memory registrations not supported.\n");
- goto out_free_mr;
+ goto out_free_pd;
}
list_add(&ndev->entry, &device_list);
@@ -469,9 +458,6 @@ out_unlock:
mutex_unlock(&device_list_mutex);
return ndev;
-out_free_mr:
- if (!register_always)
- ib_dereg_mr(ndev->mr);
out_free_pd:
ib_dealloc_pd(ndev->pd);
out_free_dev:
@@ -561,7 +547,6 @@ static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
queue = &ctrl->queues[idx];
queue->ctrl = ctrl;
- queue->flags = 0;
init_completion(&queue->cm_done);
if (idx > 0)
@@ -595,6 +580,7 @@ static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
goto out_destroy_cm_id;
}
+ clear_bit(NVME_RDMA_Q_DELETING, &queue->flags);
set_bit(NVME_RDMA_Q_CONNECTED, &queue->flags);
return 0;
@@ -639,10 +625,18 @@ static int nvme_rdma_connect_io_queues(struct nvme_rdma_ctrl *ctrl)
for (i = 1; i < ctrl->queue_count; i++) {
ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
- if (ret)
- break;
+ if (ret) {
+ dev_info(ctrl->ctrl.device,
+ "failed to connect i/o queue: %d\n", ret);
+ goto out_free_queues;
+ }
+ set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
}
+ return 0;
+
+out_free_queues:
+ nvme_rdma_free_io_queues(ctrl);
return ret;
}
@@ -727,6 +721,8 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
if (ret)
goto stop_admin_q;
+ set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
+
ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
if (ret)
goto stop_admin_q;
@@ -776,8 +772,10 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)
nvme_stop_keep_alive(&ctrl->ctrl);
- for (i = 0; i < ctrl->queue_count; i++)
+ for (i = 0; i < ctrl->queue_count; i++) {
clear_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[i].flags);
+ clear_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
+ }
if (ctrl->queue_count > 1)
nvme_stop_queues(&ctrl->ctrl);
@@ -915,7 +913,7 @@ static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
- put_unaligned_le32(queue->device->mr->rkey, sg->key);
+ put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
return 0;
}
@@ -1000,7 +998,7 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
nvme_rdma_queue_idx(queue))
return nvme_rdma_map_sg_inline(queue, req, c);
- if (!register_always)
+ if (dev->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)
return nvme_rdma_map_sg_single(queue, req, c);
}
@@ -1393,6 +1391,24 @@ nvme_rdma_timeout(struct request *rq, bool reserved)
return BLK_EH_HANDLED;
}
+/*
+ * We cannot accept any other command until the Connect command has completed.
+ */
+static inline bool nvme_rdma_queue_is_ready(struct nvme_rdma_queue *queue,
+ struct request *rq)
+{
+ if (unlikely(!test_bit(NVME_RDMA_Q_LIVE, &queue->flags))) {
+ struct nvme_command *cmd = (struct nvme_command *)rq->cmd;
+
+ if (rq->cmd_type != REQ_TYPE_DRV_PRIV ||
+ cmd->common.opcode != nvme_fabrics_command ||
+ cmd->fabrics.fctype != nvme_fabrics_type_connect)
+ return false;
+ }
+
+ return true;
+}
+
static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *bd)
{
@@ -1409,6 +1425,9 @@ static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
WARN_ON_ONCE(rq->tag < 0);
+ if (!nvme_rdma_queue_is_ready(queue, rq))
+ return BLK_MQ_RQ_QUEUE_BUSY;
+
dev = queue->device->dev;
ib_dma_sync_single_for_cpu(dev, sqe->dma,
sizeof(struct nvme_command), DMA_TO_DEVICE);
@@ -1495,7 +1514,6 @@ static void nvme_rdma_complete_rq(struct request *rq)
static struct blk_mq_ops nvme_rdma_mq_ops = {
.queue_rq = nvme_rdma_queue_rq,
.complete = nvme_rdma_complete_rq,
- .map_queue = blk_mq_map_queue,
.init_request = nvme_rdma_init_request,
.exit_request = nvme_rdma_exit_request,
.reinit_request = nvme_rdma_reinit_request,
@@ -1507,7 +1525,6 @@ static struct blk_mq_ops nvme_rdma_mq_ops = {
static struct blk_mq_ops nvme_rdma_admin_mq_ops = {
.queue_rq = nvme_rdma_queue_rq,
.complete = nvme_rdma_complete_rq,
- .map_queue = blk_mq_map_queue,
.init_request = nvme_rdma_init_admin_request,
.exit_request = nvme_rdma_exit_admin_request,
.reinit_request = nvme_rdma_reinit_request,
@@ -1561,6 +1578,8 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl)
if (error)
goto out_cleanup_queue;
+ set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
+
error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->cap);
if (error) {
dev_err(ctrl->ctrl.device,
OpenPOWER on IntegriCloud