From c56b4ef1cf0c2a2ee2593ea6f3c0b3198af11f48 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Wed, 20 Mar 2019 22:08:12 +0800 Subject: ntb: intel: Make intel_ntb3_peer_db_addr static Fix sparse warning: drivers/ntb/hw/intel/ntb_hw_gen3.c:535:5: warning: symbol 'intel_ntb3_peer_db_addr' was not declared. Should it be static? Signed-off-by: YueHaibing Acked-by: Dave Jiang Signed-off-by: Jon Mason --- drivers/ntb/hw/intel/ntb_hw_gen3.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.c b/drivers/ntb/hw/intel/ntb_hw_gen3.c index f475b56a3f49..c3397160db7f 100644 --- a/drivers/ntb/hw/intel/ntb_hw_gen3.c +++ b/drivers/ntb/hw/intel/ntb_hw_gen3.c @@ -532,9 +532,9 @@ static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, return 0; } -int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, - resource_size_t *db_size, - u64 *db_data, int db_bit) +static int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, + resource_size_t *db_size, + u64 *db_data, int db_bit) { phys_addr_t db_addr_base; struct intel_ntb_dev *ndev = ntb_ndev(ntb); -- cgit v1.2.1 From 8b2f033631c2ec735c10846030fad44f3bfdaf07 Mon Sep 17 00:00:00 2001 From: Sanjay R Mehta Date: Fri, 15 Feb 2019 09:20:07 +0000 Subject: NTB: ntb_perf: Increased the number of message retries to 1000 while waiting for the peer ntb_perf to initialize scratchpad registers, local side ntb_perf might have already exhausted the maximum number of retries which is currently set to 500. To avoid this and to give little more time to the peer ntb_perf for scratchpad initialization, increased the number of retries to 1000 Signed-off-by: Sanjay R Mehta Acked-by: Allen Hubbe Acked-by: Logan Gunthorpe Acked-by: Serge Semin Signed-off-by: Jon Mason --- drivers/ntb/test/ntb_perf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c index 11a6cd374004..5ed84376ae56 100644 --- a/drivers/ntb/test/ntb_perf.c +++ b/drivers/ntb/test/ntb_perf.c @@ -100,7 +100,7 @@ MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool"); #define DMA_TRIES 100 #define DMA_MDELAY 10 -#define MSG_TRIES 500 +#define MSG_TRIES 1000 #define MSG_UDELAY_LOW 1000 #define MSG_UDELAY_HIGH 2000 -- cgit v1.2.1 From b1ee5998a5d2bb16fb2bb9a03a30a83a55012b92 Mon Sep 17 00:00:00 2001 From: Sanjay R Mehta Date: Fri, 15 Feb 2019 09:20:30 +0000 Subject: NTB: ntb_perf: Disable NTB link after clearing peer XLAT registers If ntb link disabled before clearing peer's XLAT register, the clearing won't have any effect since the link is already down. So modified the sequence so that the link is down only towards the end of the function after clearing the XLAT register Signed-off-by: Sanjay R Mehta Acked-by: Allen Hubbe Acked-by: Logan Gunthorpe Acked-by: Serge Semin Signed-off-by: Jon Mason --- drivers/ntb/test/ntb_perf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c index 5ed84376ae56..32a70b6491cd 100644 --- a/drivers/ntb/test/ntb_perf.c +++ b/drivers/ntb/test/ntb_perf.c @@ -734,8 +734,6 @@ static void perf_disable_service(struct perf_ctx *perf) { int pidx; - ntb_link_disable(perf->ntb); - if (perf->cmd_send == perf_msg_cmd_send) { u64 inbits; @@ -752,6 +750,8 @@ static void perf_disable_service(struct perf_ctx *perf) for (pidx = 0; pidx < perf->pcnt; pidx++) flush_work(&perf->peers[pidx].service); + + ntb_link_disable(perf->ntb); } /*============================================================================== -- cgit v1.2.1 From 12c023d7c32ae14254c8903f0466d37dea6d21ed Mon Sep 17 00:00:00 2001 From: Sanjay R Mehta Date: Fri, 15 Feb 2019 09:21:19 +0000 Subject: NTB: ntb_perf: Clear stale values in doorbell and command SPAD register when ntb_perf is unloaded, the command scratchpad register still retains the last initialized value of PERF_CMD_INVAL. When ntb_perf is re-loaded and reads peer command scratchpad register and it mis interprets the peer state as initialized. To avoid this, clearing the local side command scratchpad register in perf_disable_service Signed-off-by: Sanjay R Mehta Acked-by: Allen Hubbe Acked-by: Logan Gunthorpe Acked-by: Serge Semin Signed-off-by: Jon Mason --- drivers/ntb/test/ntb_perf.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/ntb') diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c index 32a70b6491cd..d028331558ea 100644 --- a/drivers/ntb/test/ntb_perf.c +++ b/drivers/ntb/test/ntb_perf.c @@ -751,6 +751,14 @@ static void perf_disable_service(struct perf_ctx *perf) for (pidx = 0; pidx < perf->pcnt; pidx++) flush_work(&perf->peers[pidx].service); + for (pidx = 0; pidx < perf->pcnt; pidx++) { + struct perf_peer *peer = &perf->peers[pidx]; + + ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx), 0); + } + + ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx)); + ntb_link_disable(perf->ntb); } -- cgit v1.2.1 From a5c3666f1c32714d8ee8735a129a9f13773135ba Mon Sep 17 00:00:00 2001 From: Sanjay R Mehta Date: Fri, 15 Feb 2019 09:21:46 +0000 Subject: NTB: ntb_hw_amd: set peer limit register As per amd ntb spec it says that peer limit register must be programmed Signed-off-by: Sanjay R Mehta Acked-by: Allen Hubbe Acked-by: Logan Gunthorpe Acked-by: Serge Semin Signed-off-by: Jon Mason --- drivers/ntb/hw/amd/ntb_hw_amd.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c index efb214fc545a..cd463e132035 100644 --- a/drivers/ntb/hw/amd/ntb_hw_amd.c +++ b/drivers/ntb/hw/amd/ntb_hw_amd.c @@ -160,8 +160,8 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, } /* set and verify setting the limit */ - write64(limit, mmio + limit_reg); - reg_val = read64(mmio + limit_reg); + write64(limit, peer_mmio + limit_reg); + reg_val = read64(peer_mmio + limit_reg); if (reg_val != limit) { write64(base_addr, mmio + limit_reg); write64(0, peer_mmio + xlat_reg); @@ -183,8 +183,8 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, } /* set and verify setting the limit */ - writel(limit, mmio + limit_reg); - reg_val = readl(mmio + limit_reg); + writel(limit, peer_mmio + limit_reg); + reg_val = readl(peer_mmio + limit_reg); if (reg_val != limit) { writel(base_addr, mmio + limit_reg); writel(0, peer_mmio + xlat_reg); -- cgit v1.2.1 From 51cb8dbf13246a0b144bff01d7d617efe550a055 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Tue, 19 Feb 2019 12:56:36 -0700 Subject: NTB: ntb_transport: Ensure qp->tx_mw_dma_addr is initaliazed Dan Carpenter's static checker reported: drivers/ntb/ntb_transport.c:1926 ntb_transport_create_queue() error: we previously assumed 'qp->tx_dma_chan' could be null (see line 1872) This is because the tx_mw_dma_addr is uninitialized in this function and may be incorrectly released using a NULL DMA channel. In practice this bug will not likely be seen. I'd guess you could hit this if you loaded ntb_netdev with use_dma=True, then unloaded it and loaded it again after setting the module parameter to use_dma=False. To fix this, we simply ensure that tx_mw_dma_addr is always initialized to zero. This is the safest in case any other part of the code operates on it if it is non-zero. Fixes: c59666bb32b9 ("NTB: ntb_transport: Ensure the destination buffer is mapped for TX DMA") Signed-off-by: Logan Gunthorpe Cc: Dan Carpenter Signed-off-by: Jon Mason --- drivers/ntb/ntb_transport.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/ntb') diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index d4f39ba1d976..99033228dccb 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -1869,6 +1869,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev, qp->rx_dma_chan = NULL; } + qp->tx_mw_dma_addr = 0; if (qp->tx_dma_chan) { qp->tx_mw_dma_addr = dma_map_resource(qp->tx_dma_chan->device->dev, -- cgit v1.2.1 From ff148d8ac53e59802645bd3200c811620317eb9f Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 25 Mar 2019 12:17:26 +0300 Subject: ntb_hw_switchtec: potential shift wrapping bug in switchtec_ntb_init_sndev() This code triggers a Smatch warning: drivers/ntb/hw/mscc/ntb_hw_switchtec.c:884 switchtec_ntb_init_sndev() warn: should '(1 << sndev->peer_partition)' be a 64 bit type? The "part_map" and "tpart_vec" variables are u64 type so this seems like a valid warning. Fixes: 3df54c870f52 ("ntb_hw_switchtec: Allow using Switchtec NTB in multi-partition setups") Signed-off-by: Dan Carpenter Reviewed-by: Logan Gunthorpe Signed-off-by: Jon Mason --- drivers/ntb/hw/mscc/ntb_hw_switchtec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index d905d368d28c..26a88731b0cb 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -881,7 +881,7 @@ static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) } sndev->peer_partition = ffs(tpart_vec) - 1; - if (!(part_map & (1 << sndev->peer_partition))) { + if (!(part_map & (1ULL << sndev->peer_partition))) { dev_err(&sndev->stdev->dev, "ntb target partition is not NT partition\n"); return -ENODEV; -- cgit v1.2.1 From 1e590dec3ed6852dba6fcbcc7e5300c893914fd7 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 25 Mar 2019 12:23:26 +0300 Subject: NTB: amd: Silence shift wrapping warning in amd_ntb_db_vector_mask() This code triggers a Smatch warning: drivers/ntb/hw/amd/ntb_hw_amd.c:336 amd_ntb_db_vector_mask() warn: should '(1 << db_vector)' be a 64 bit type? I don't think "db_vector" can be higher than 16 so this doesn't affect runtime, but it's nice to silence the static checker warning and we might increase "ndev->db_count" in the future. Signed-off-by: Dan Carpenter Acked-by: Shyam Sundar S K Signed-off-by: Jon Mason --- drivers/ntb/hw/amd/ntb_hw_amd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c index cd463e132035..2859cc99b73e 100644 --- a/drivers/ntb/hw/amd/ntb_hw_amd.c +++ b/drivers/ntb/hw/amd/ntb_hw_amd.c @@ -333,7 +333,7 @@ static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) if (db_vector < 0 || db_vector > ndev->db_count) return 0; - return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector); + return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector); } static u64 amd_ntb_db_read(struct ntb_dev *ntb) -- cgit v1.2.1 From 28e339f100e34608a8f40c5ec6a6f8ef7b826028 Mon Sep 17 00:00:00 2001 From: Joey Zhang Date: Thu, 6 Jun 2019 15:09:42 +0800 Subject: ntb_hw_switchtec: Remove redundant steps of switchtec_ntb_reinit_peer() function When a re-initialization is caused by a link event, the driver will re-setup the shared memory window. But at that time, the shared memory is still valid, and it's unnecessary to free, reallocate and then initialize it again. We only need to reconfigure the hardware registers. Remove the redundant steps from switchtec_ntb_reinit_peer() function. Signed-off-by: Joey Zhang Signed-off-by: Wesley Sheng Reviewed-by: Logan Gunthorpe Signed-off-by: Jon Mason --- drivers/ntb/hw/mscc/ntb_hw_switchtec.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index 26a88731b0cb..c408b7adeaa6 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -1457,10 +1457,13 @@ static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) { - dev_info(&sndev->stdev->dev, "peer reinitialized\n"); - switchtec_ntb_deinit_shared_mw(sndev); - switchtec_ntb_init_mw(sndev); - return switchtec_ntb_init_shared_mw(sndev); + int rc; + + dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n"); + rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, + sndev->self_partition, + sndev->self_shared_dma); + return rc; } static int switchtec_ntb_add(struct device *dev, -- cgit v1.2.1 From f0f43e766bdb314b3816ac891fb18cea5de4123d Mon Sep 17 00:00:00 2001 From: Wesley Sheng Date: Thu, 6 Jun 2019 15:09:43 +0800 Subject: ntb_hw_switchtec: Skip unnecessary re-setup of shared memory window for crosslink case In case of NTB crosslink topology, the setting of shared memory window in the virtual partition doesn't reset on peer's reboot. So skip the unnecessary re-setup of shared memory window for that case. Signed-off-by: Wesley Sheng Signed-off-by: Jon Mason --- drivers/ntb/hw/mscc/ntb_hw_switchtec.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/ntb') diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index c408b7adeaa6..a26e3d0cf457 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -1459,6 +1459,9 @@ static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) { int rc; + if (crosslink_is_enabled(sndev)) + return 0; + dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n"); rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, sndev->self_partition, -- cgit v1.2.1 From a944ccc3b0aedb7f3a24b8c90624fe182e8a8851 Mon Sep 17 00:00:00 2001 From: Joey Zhang Date: Thu, 6 Jun 2019 15:09:44 +0800 Subject: ntb_hw_switchtec: Fix setup MW with failure bug Switchtec does not support setting multiple MWs simultaneously. The driver takes a hardware lock to ensure that two peers are not doing this simultaneously and it fails if someone else takes the lock. In most cases, this is fine as clients only setup the MWs once on one side of the link. However, there's a race condition when a re-initialization is caused by a link event. The driver will re-setup the shared memory window asynchronously and this races with the client setting up it's memory windows on the link up event. To fix this we ensure do the entire initialization in a work queue and signal the client once it's done. Signed-off-by: Joey Zhang Signed-off-by: Wesley Sheng Signed-off-by: Jon Mason --- drivers/ntb/hw/mscc/ntb_hw_switchtec.c | 66 ++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 27 deletions(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index a26e3d0cf457..7e3e16c04eef 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -95,7 +95,8 @@ struct switchtec_ntb { bool link_is_up; enum ntb_speed link_speed; enum ntb_width link_width; - struct work_struct link_reinit_work; + struct work_struct check_link_status_work; + bool link_force_down; }; static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) @@ -494,33 +495,11 @@ enum switchtec_msg { static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev); -static void link_reinit_work(struct work_struct *work) -{ - struct switchtec_ntb *sndev; - - sndev = container_of(work, struct switchtec_ntb, link_reinit_work); - - switchtec_ntb_reinit_peer(sndev); -} - -static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, - enum switchtec_msg msg) +static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev) { int link_sta; int old = sndev->link_is_up; - if (msg == MSG_LINK_FORCE_DOWN) { - schedule_work(&sndev->link_reinit_work); - - if (sndev->link_is_up) { - sndev->link_is_up = 0; - ntb_link_event(&sndev->ntb); - dev_info(&sndev->stdev->dev, "ntb link forced down\n"); - } - - return; - } - link_sta = sndev->self_shared->link_sta; if (link_sta) { u64 peer = ioread64(&sndev->peer_shared->magic); @@ -545,6 +524,38 @@ static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, } } +static void check_link_status_work(struct work_struct *work) +{ + struct switchtec_ntb *sndev; + + sndev = container_of(work, struct switchtec_ntb, + check_link_status_work); + + if (sndev->link_force_down) { + sndev->link_force_down = false; + switchtec_ntb_reinit_peer(sndev); + + if (sndev->link_is_up) { + sndev->link_is_up = 0; + ntb_link_event(&sndev->ntb); + dev_info(&sndev->stdev->dev, "ntb link forced down\n"); + } + + return; + } + + switchtec_ntb_link_status_update(sndev); +} + +static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, + enum switchtec_msg msg) +{ + if (msg == MSG_LINK_FORCE_DOWN) + sndev->link_force_down = true; + + schedule_work(&sndev->check_link_status_work); +} + static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) { struct switchtec_ntb *sndev = stdev->sndev; @@ -577,7 +588,7 @@ static int switchtec_ntb_link_enable(struct ntb_dev *ntb, sndev->self_shared->link_sta = 1; switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); - switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); + switchtec_ntb_link_status_update(sndev); return 0; } @@ -591,7 +602,7 @@ static int switchtec_ntb_link_disable(struct ntb_dev *ntb) sndev->self_shared->link_sta = 0; switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN); - switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); + switchtec_ntb_link_status_update(sndev); return 0; } @@ -844,7 +855,8 @@ static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) sndev->ntb.topo = NTB_TOPO_SWITCH; sndev->ntb.ops = &switchtec_ntb_ops; - INIT_WORK(&sndev->link_reinit_work, link_reinit_work); + INIT_WORK(&sndev->check_link_status_work, check_link_status_work); + sndev->link_force_down = false; sndev->self_partition = sndev->stdev->partition; -- cgit v1.2.1 From d217e07b32a6750e44f529e0218898c024b2c637 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Thu, 23 May 2019 16:30:55 -0600 Subject: NTB: Rename ntb.c to support multiple source files in the module The kbuild system does not support having multiple source files in a module if one of those source files has the same name as the module. Therefore, we must rename ntb.c to core.c, while the module remains ntb.ko. This is similar to the way the nvme modules are structured. Signed-off-by: Logan Gunthorpe Cc: Dave Jiang Cc: Allen Hubbe Signed-off-by: Jon Mason --- drivers/ntb/Makefile | 2 + drivers/ntb/core.c | 318 +++++++++++++++++++++++++++++++++++++++++++++++++++ drivers/ntb/ntb.c | 318 --------------------------------------------------- 3 files changed, 320 insertions(+), 318 deletions(-) create mode 100644 drivers/ntb/core.c delete mode 100644 drivers/ntb/ntb.c (limited to 'drivers/ntb') diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile index 1921dec1949d..537226f8e78d 100644 --- a/drivers/ntb/Makefile +++ b/drivers/ntb/Makefile @@ -1,2 +1,4 @@ obj-$(CONFIG_NTB) += ntb.o hw/ test/ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o + +ntb-y := core.o diff --git a/drivers/ntb/core.c b/drivers/ntb/core.c new file mode 100644 index 000000000000..2581ab724c34 --- /dev/null +++ b/drivers/ntb/core.c @@ -0,0 +1,318 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * Copyright (C) 2016 T-Platforms. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * BSD LICENSE + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * Copyright (C) 2016 T-Platforms. All Rights Reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * PCIe NTB Linux driver + * + * Contact Information: + * Allen Hubbe + */ + +#include +#include +#include + +#include +#include + +#define DRIVER_NAME "ntb" +#define DRIVER_DESCRIPTION "PCIe NTB Driver Framework" + +#define DRIVER_VERSION "1.0" +#define DRIVER_RELDATE "24 March 2015" +#define DRIVER_AUTHOR "Allen Hubbe " + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_VERSION(DRIVER_VERSION); +MODULE_AUTHOR(DRIVER_AUTHOR); +MODULE_DESCRIPTION(DRIVER_DESCRIPTION); + +static struct bus_type ntb_bus; +static void ntb_dev_release(struct device *dev); + +int __ntb_register_client(struct ntb_client *client, struct module *mod, + const char *mod_name) +{ + if (!client) + return -EINVAL; + if (!ntb_client_ops_is_valid(&client->ops)) + return -EINVAL; + + memset(&client->drv, 0, sizeof(client->drv)); + client->drv.bus = &ntb_bus; + client->drv.name = mod_name; + client->drv.owner = mod; + + return driver_register(&client->drv); +} +EXPORT_SYMBOL(__ntb_register_client); + +void ntb_unregister_client(struct ntb_client *client) +{ + driver_unregister(&client->drv); +} +EXPORT_SYMBOL(ntb_unregister_client); + +int ntb_register_device(struct ntb_dev *ntb) +{ + if (!ntb) + return -EINVAL; + if (!ntb->pdev) + return -EINVAL; + if (!ntb->ops) + return -EINVAL; + if (!ntb_dev_ops_is_valid(ntb->ops)) + return -EINVAL; + + init_completion(&ntb->released); + + ntb->dev.bus = &ntb_bus; + ntb->dev.parent = &ntb->pdev->dev; + ntb->dev.release = ntb_dev_release; + dev_set_name(&ntb->dev, "%s", pci_name(ntb->pdev)); + + ntb->ctx = NULL; + ntb->ctx_ops = NULL; + spin_lock_init(&ntb->ctx_lock); + + return device_register(&ntb->dev); +} +EXPORT_SYMBOL(ntb_register_device); + +void ntb_unregister_device(struct ntb_dev *ntb) +{ + device_unregister(&ntb->dev); + wait_for_completion(&ntb->released); +} +EXPORT_SYMBOL(ntb_unregister_device); + +int ntb_set_ctx(struct ntb_dev *ntb, void *ctx, + const struct ntb_ctx_ops *ctx_ops) +{ + unsigned long irqflags; + + if (!ntb_ctx_ops_is_valid(ctx_ops)) + return -EINVAL; + if (ntb->ctx_ops) + return -EINVAL; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + ntb->ctx = ctx; + ntb->ctx_ops = ctx_ops; + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); + + return 0; +} +EXPORT_SYMBOL(ntb_set_ctx); + +void ntb_clear_ctx(struct ntb_dev *ntb) +{ + unsigned long irqflags; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + ntb->ctx_ops = NULL; + ntb->ctx = NULL; + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); +} +EXPORT_SYMBOL(ntb_clear_ctx); + +void ntb_link_event(struct ntb_dev *ntb) +{ + unsigned long irqflags; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + if (ntb->ctx_ops && ntb->ctx_ops->link_event) + ntb->ctx_ops->link_event(ntb->ctx); + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); +} +EXPORT_SYMBOL(ntb_link_event); + +void ntb_db_event(struct ntb_dev *ntb, int vector) +{ + unsigned long irqflags; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + if (ntb->ctx_ops && ntb->ctx_ops->db_event) + ntb->ctx_ops->db_event(ntb->ctx, vector); + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); +} +EXPORT_SYMBOL(ntb_db_event); + +void ntb_msg_event(struct ntb_dev *ntb) +{ + unsigned long irqflags; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + if (ntb->ctx_ops && ntb->ctx_ops->msg_event) + ntb->ctx_ops->msg_event(ntb->ctx); + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); +} +EXPORT_SYMBOL(ntb_msg_event); + +int ntb_default_port_number(struct ntb_dev *ntb) +{ + switch (ntb->topo) { + case NTB_TOPO_PRI: + case NTB_TOPO_B2B_USD: + return NTB_PORT_PRI_USD; + case NTB_TOPO_SEC: + case NTB_TOPO_B2B_DSD: + return NTB_PORT_SEC_DSD; + default: + break; + } + + return -EINVAL; +} +EXPORT_SYMBOL(ntb_default_port_number); + +int ntb_default_peer_port_count(struct ntb_dev *ntb) +{ + return NTB_DEF_PEER_CNT; +} +EXPORT_SYMBOL(ntb_default_peer_port_count); + +int ntb_default_peer_port_number(struct ntb_dev *ntb, int pidx) +{ + if (pidx != NTB_DEF_PEER_IDX) + return -EINVAL; + + switch (ntb->topo) { + case NTB_TOPO_PRI: + case NTB_TOPO_B2B_USD: + return NTB_PORT_SEC_DSD; + case NTB_TOPO_SEC: + case NTB_TOPO_B2B_DSD: + return NTB_PORT_PRI_USD; + default: + break; + } + + return -EINVAL; +} +EXPORT_SYMBOL(ntb_default_peer_port_number); + +int ntb_default_peer_port_idx(struct ntb_dev *ntb, int port) +{ + int peer_port = ntb_default_peer_port_number(ntb, NTB_DEF_PEER_IDX); + + if (peer_port == -EINVAL || port != peer_port) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL(ntb_default_peer_port_idx); + +static int ntb_probe(struct device *dev) +{ + struct ntb_dev *ntb; + struct ntb_client *client; + int rc; + + get_device(dev); + ntb = dev_ntb(dev); + client = drv_ntb_client(dev->driver); + + rc = client->ops.probe(client, ntb); + if (rc) + put_device(dev); + + return rc; +} + +static int ntb_remove(struct device *dev) +{ + struct ntb_dev *ntb; + struct ntb_client *client; + + if (dev->driver) { + ntb = dev_ntb(dev); + client = drv_ntb_client(dev->driver); + + client->ops.remove(client, ntb); + put_device(dev); + } + + return 0; +} + +static void ntb_dev_release(struct device *dev) +{ + struct ntb_dev *ntb = dev_ntb(dev); + + complete(&ntb->released); +} + +static struct bus_type ntb_bus = { + .name = "ntb", + .probe = ntb_probe, + .remove = ntb_remove, +}; + +static int __init ntb_driver_init(void) +{ + return bus_register(&ntb_bus); +} +module_init(ntb_driver_init); + +static void __exit ntb_driver_exit(void) +{ + bus_unregister(&ntb_bus); +} +module_exit(ntb_driver_exit); + diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c deleted file mode 100644 index 2581ab724c34..000000000000 --- a/drivers/ntb/ntb.c +++ /dev/null @@ -1,318 +0,0 @@ -/* - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright (C) 2015 EMC Corporation. All Rights Reserved. - * Copyright (C) 2016 T-Platforms. All Rights Reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * BSD LICENSE - * - * Copyright (C) 2015 EMC Corporation. All Rights Reserved. - * Copyright (C) 2016 T-Platforms. All Rights Reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copy - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * PCIe NTB Linux driver - * - * Contact Information: - * Allen Hubbe - */ - -#include -#include -#include - -#include -#include - -#define DRIVER_NAME "ntb" -#define DRIVER_DESCRIPTION "PCIe NTB Driver Framework" - -#define DRIVER_VERSION "1.0" -#define DRIVER_RELDATE "24 March 2015" -#define DRIVER_AUTHOR "Allen Hubbe " - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_VERSION(DRIVER_VERSION); -MODULE_AUTHOR(DRIVER_AUTHOR); -MODULE_DESCRIPTION(DRIVER_DESCRIPTION); - -static struct bus_type ntb_bus; -static void ntb_dev_release(struct device *dev); - -int __ntb_register_client(struct ntb_client *client, struct module *mod, - const char *mod_name) -{ - if (!client) - return -EINVAL; - if (!ntb_client_ops_is_valid(&client->ops)) - return -EINVAL; - - memset(&client->drv, 0, sizeof(client->drv)); - client->drv.bus = &ntb_bus; - client->drv.name = mod_name; - client->drv.owner = mod; - - return driver_register(&client->drv); -} -EXPORT_SYMBOL(__ntb_register_client); - -void ntb_unregister_client(struct ntb_client *client) -{ - driver_unregister(&client->drv); -} -EXPORT_SYMBOL(ntb_unregister_client); - -int ntb_register_device(struct ntb_dev *ntb) -{ - if (!ntb) - return -EINVAL; - if (!ntb->pdev) - return -EINVAL; - if (!ntb->ops) - return -EINVAL; - if (!ntb_dev_ops_is_valid(ntb->ops)) - return -EINVAL; - - init_completion(&ntb->released); - - ntb->dev.bus = &ntb_bus; - ntb->dev.parent = &ntb->pdev->dev; - ntb->dev.release = ntb_dev_release; - dev_set_name(&ntb->dev, "%s", pci_name(ntb->pdev)); - - ntb->ctx = NULL; - ntb->ctx_ops = NULL; - spin_lock_init(&ntb->ctx_lock); - - return device_register(&ntb->dev); -} -EXPORT_SYMBOL(ntb_register_device); - -void ntb_unregister_device(struct ntb_dev *ntb) -{ - device_unregister(&ntb->dev); - wait_for_completion(&ntb->released); -} -EXPORT_SYMBOL(ntb_unregister_device); - -int ntb_set_ctx(struct ntb_dev *ntb, void *ctx, - const struct ntb_ctx_ops *ctx_ops) -{ - unsigned long irqflags; - - if (!ntb_ctx_ops_is_valid(ctx_ops)) - return -EINVAL; - if (ntb->ctx_ops) - return -EINVAL; - - spin_lock_irqsave(&ntb->ctx_lock, irqflags); - { - ntb->ctx = ctx; - ntb->ctx_ops = ctx_ops; - } - spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); - - return 0; -} -EXPORT_SYMBOL(ntb_set_ctx); - -void ntb_clear_ctx(struct ntb_dev *ntb) -{ - unsigned long irqflags; - - spin_lock_irqsave(&ntb->ctx_lock, irqflags); - { - ntb->ctx_ops = NULL; - ntb->ctx = NULL; - } - spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); -} -EXPORT_SYMBOL(ntb_clear_ctx); - -void ntb_link_event(struct ntb_dev *ntb) -{ - unsigned long irqflags; - - spin_lock_irqsave(&ntb->ctx_lock, irqflags); - { - if (ntb->ctx_ops && ntb->ctx_ops->link_event) - ntb->ctx_ops->link_event(ntb->ctx); - } - spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); -} -EXPORT_SYMBOL(ntb_link_event); - -void ntb_db_event(struct ntb_dev *ntb, int vector) -{ - unsigned long irqflags; - - spin_lock_irqsave(&ntb->ctx_lock, irqflags); - { - if (ntb->ctx_ops && ntb->ctx_ops->db_event) - ntb->ctx_ops->db_event(ntb->ctx, vector); - } - spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); -} -EXPORT_SYMBOL(ntb_db_event); - -void ntb_msg_event(struct ntb_dev *ntb) -{ - unsigned long irqflags; - - spin_lock_irqsave(&ntb->ctx_lock, irqflags); - { - if (ntb->ctx_ops && ntb->ctx_ops->msg_event) - ntb->ctx_ops->msg_event(ntb->ctx); - } - spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); -} -EXPORT_SYMBOL(ntb_msg_event); - -int ntb_default_port_number(struct ntb_dev *ntb) -{ - switch (ntb->topo) { - case NTB_TOPO_PRI: - case NTB_TOPO_B2B_USD: - return NTB_PORT_PRI_USD; - case NTB_TOPO_SEC: - case NTB_TOPO_B2B_DSD: - return NTB_PORT_SEC_DSD; - default: - break; - } - - return -EINVAL; -} -EXPORT_SYMBOL(ntb_default_port_number); - -int ntb_default_peer_port_count(struct ntb_dev *ntb) -{ - return NTB_DEF_PEER_CNT; -} -EXPORT_SYMBOL(ntb_default_peer_port_count); - -int ntb_default_peer_port_number(struct ntb_dev *ntb, int pidx) -{ - if (pidx != NTB_DEF_PEER_IDX) - return -EINVAL; - - switch (ntb->topo) { - case NTB_TOPO_PRI: - case NTB_TOPO_B2B_USD: - return NTB_PORT_SEC_DSD; - case NTB_TOPO_SEC: - case NTB_TOPO_B2B_DSD: - return NTB_PORT_PRI_USD; - default: - break; - } - - return -EINVAL; -} -EXPORT_SYMBOL(ntb_default_peer_port_number); - -int ntb_default_peer_port_idx(struct ntb_dev *ntb, int port) -{ - int peer_port = ntb_default_peer_port_number(ntb, NTB_DEF_PEER_IDX); - - if (peer_port == -EINVAL || port != peer_port) - return -EINVAL; - - return 0; -} -EXPORT_SYMBOL(ntb_default_peer_port_idx); - -static int ntb_probe(struct device *dev) -{ - struct ntb_dev *ntb; - struct ntb_client *client; - int rc; - - get_device(dev); - ntb = dev_ntb(dev); - client = drv_ntb_client(dev->driver); - - rc = client->ops.probe(client, ntb); - if (rc) - put_device(dev); - - return rc; -} - -static int ntb_remove(struct device *dev) -{ - struct ntb_dev *ntb; - struct ntb_client *client; - - if (dev->driver) { - ntb = dev_ntb(dev); - client = drv_ntb_client(dev->driver); - - client->ops.remove(client, ntb); - put_device(dev); - } - - return 0; -} - -static void ntb_dev_release(struct device *dev) -{ - struct ntb_dev *ntb = dev_ntb(dev); - - complete(&ntb->released); -} - -static struct bus_type ntb_bus = { - .name = "ntb", - .probe = ntb_probe, - .remove = ntb_remove, -}; - -static int __init ntb_driver_init(void) -{ - return bus_register(&ntb_bus); -} -module_init(ntb_driver_init); - -static void __exit ntb_driver_exit(void) -{ - bus_unregister(&ntb_bus); -} -module_exit(ntb_driver_exit); - -- cgit v1.2.1 From 26b3a37b928457ba2cd98eaf6d7b0feca5a30fa6 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Thu, 23 May 2019 16:30:56 -0600 Subject: NTB: Introduce MSI library The NTB MSI library allows passing MSI interrupts across a memory window. This offers similar functionality to doorbells or messages except will often have much better latency and the client can potentially use significantly more remote interrupts than typical hardware provides for doorbells. (Which can be important in high-multiport setups.) The library utilizes one memory window per peer and uses the highest index memory windows. Before any ntb_msi function may be used, the user must call ntb_msi_init(). It may then setup and tear down the memory windows when the link state changes using ntb_msi_setup_mws() and ntb_msi_clear_mws(). The peer which receives the interrupt must call ntb_msim_request_irq() to assign the interrupt handler (this function is functionally similar to devm_request_irq()) and the returned descriptor must be transferred to the peer which can use it to trigger the interrupt. The triggering peer, once having received the descriptor, can trigger the interrupt by calling ntb_msi_peer_trigger(). Signed-off-by: Logan Gunthorpe Cc: Dave Jiang Cc: Allen Hubbe Signed-off-by: Jon Mason --- drivers/ntb/Kconfig | 11 ++ drivers/ntb/Makefile | 3 +- drivers/ntb/msi.c | 415 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 drivers/ntb/msi.c (limited to 'drivers/ntb') diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig index 95944e52fa36..5760764052be 100644 --- a/drivers/ntb/Kconfig +++ b/drivers/ntb/Kconfig @@ -12,6 +12,17 @@ menuconfig NTB if NTB +config NTB_MSI + bool "MSI Interrupt Support" + depends on PCI_MSI + help + Support using MSI interrupt forwarding instead of (or in addition to) + hardware doorbells. MSI interrupts typically offer lower latency + than doorbells and more MSI interrupts can be made available to + clients. However this requires an extra memory window and support + in the hardware driver for creating the MSI interrupts. + + If unsure, say N. source "drivers/ntb/hw/Kconfig" source "drivers/ntb/test/Kconfig" diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile index 537226f8e78d..cc27ad2ef150 100644 --- a/drivers/ntb/Makefile +++ b/drivers/ntb/Makefile @@ -1,4 +1,5 @@ obj-$(CONFIG_NTB) += ntb.o hw/ test/ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o -ntb-y := core.o +ntb-y := core.o +ntb-$(CONFIG_NTB_MSI) += msi.o diff --git a/drivers/ntb/msi.c b/drivers/ntb/msi.c new file mode 100644 index 000000000000..9dddf133658f --- /dev/null +++ b/drivers/ntb/msi.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) + +#include +#include +#include +#include +#include + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_VERSION("0.1"); +MODULE_AUTHOR("Logan Gunthorpe "); +MODULE_DESCRIPTION("NTB MSI Interrupt Library"); + +struct ntb_msi { + u64 base_addr; + u64 end_addr; + + void (*desc_changed)(void *ctx); + + u32 __iomem *peer_mws[]; +}; + +/** + * ntb_msi_init() - Initialize the MSI context + * @ntb: NTB device context + * + * This function must be called before any other ntb_msi function. + * It initializes the context for MSI operations and maps + * the peer memory windows. + * + * This function reserves the last N outbound memory windows (where N + * is the number of peers). + * + * Return: Zero on success, otherwise a negative error number. + */ +int ntb_msi_init(struct ntb_dev *ntb, + void (*desc_changed)(void *ctx)) +{ + phys_addr_t mw_phys_addr; + resource_size_t mw_size; + size_t struct_size; + int peer_widx; + int peers; + int ret; + int i; + + peers = ntb_peer_port_count(ntb); + if (peers <= 0) + return -EINVAL; + + struct_size = sizeof(*ntb->msi) + sizeof(*ntb->msi->peer_mws) * peers; + + ntb->msi = devm_kzalloc(&ntb->dev, struct_size, GFP_KERNEL); + if (!ntb->msi) + return -ENOMEM; + + ntb->msi->desc_changed = desc_changed; + + for (i = 0; i < peers; i++) { + peer_widx = ntb_peer_mw_count(ntb) - 1 - i; + + ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr, + &mw_size); + if (ret) + goto unroll; + + ntb->msi->peer_mws[i] = devm_ioremap(&ntb->dev, mw_phys_addr, + mw_size); + if (!ntb->msi->peer_mws[i]) { + ret = -EFAULT; + goto unroll; + } + } + + return 0; + +unroll: + for (i = 0; i < peers; i++) + if (ntb->msi->peer_mws[i]) + devm_iounmap(&ntb->dev, ntb->msi->peer_mws[i]); + + devm_kfree(&ntb->dev, ntb->msi); + ntb->msi = NULL; + return ret; +} +EXPORT_SYMBOL(ntb_msi_init); + +/** + * ntb_msi_setup_mws() - Initialize the MSI inbound memory windows + * @ntb: NTB device context + * + * This function sets up the required inbound memory windows. It should be + * called from a work function after a link up event. + * + * Over the entire network, this function will reserves the last N + * inbound memory windows for each peer (where N is the number of peers). + * + * ntb_msi_init() must be called before this function. + * + * Return: Zero on success, otherwise a negative error number. + */ +int ntb_msi_setup_mws(struct ntb_dev *ntb) +{ + struct msi_desc *desc; + u64 addr; + int peer, peer_widx; + resource_size_t addr_align, size_align, size_max; + resource_size_t mw_size = SZ_32K; + resource_size_t mw_min_size = mw_size; + int i; + int ret; + + if (!ntb->msi) + return -EINVAL; + + desc = first_msi_entry(&ntb->pdev->dev); + addr = desc->msg.address_lo + ((uint64_t)desc->msg.address_hi << 32); + + for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) { + peer_widx = ntb_peer_highest_mw_idx(ntb, peer); + if (peer_widx < 0) + return peer_widx; + + ret = ntb_mw_get_align(ntb, peer, peer_widx, &addr_align, + NULL, NULL); + if (ret) + return ret; + + addr &= ~(addr_align - 1); + } + + for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) { + peer_widx = ntb_peer_highest_mw_idx(ntb, peer); + if (peer_widx < 0) { + ret = peer_widx; + goto error_out; + } + + ret = ntb_mw_get_align(ntb, peer, peer_widx, NULL, + &size_align, &size_max); + if (ret) + goto error_out; + + mw_size = round_up(mw_size, size_align); + mw_size = max(mw_size, size_max); + if (mw_size < mw_min_size) + mw_min_size = mw_size; + + ret = ntb_mw_set_trans(ntb, peer, peer_widx, + addr, mw_size); + if (ret) + goto error_out; + } + + ntb->msi->base_addr = addr; + ntb->msi->end_addr = addr + mw_min_size; + + return 0; + +error_out: + for (i = 0; i < peer; i++) { + peer_widx = ntb_peer_highest_mw_idx(ntb, peer); + if (peer_widx < 0) + continue; + + ntb_mw_clear_trans(ntb, i, peer_widx); + } + + return ret; +} +EXPORT_SYMBOL(ntb_msi_setup_mws); + +/** + * ntb_msi_clear_mws() - Clear all inbound memory windows + * @ntb: NTB device context + * + * This function tears down the resources used by ntb_msi_setup_mws(). + */ +void ntb_msi_clear_mws(struct ntb_dev *ntb) +{ + int peer; + int peer_widx; + + for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) { + peer_widx = ntb_peer_highest_mw_idx(ntb, peer); + if (peer_widx < 0) + continue; + + ntb_mw_clear_trans(ntb, peer, peer_widx); + } +} +EXPORT_SYMBOL(ntb_msi_clear_mws); + +struct ntb_msi_devres { + struct ntb_dev *ntb; + struct msi_desc *entry; + struct ntb_msi_desc *msi_desc; +}; + +static int ntb_msi_set_desc(struct ntb_dev *ntb, struct msi_desc *entry, + struct ntb_msi_desc *msi_desc) +{ + u64 addr; + + addr = entry->msg.address_lo + + ((uint64_t)entry->msg.address_hi << 32); + + if (addr < ntb->msi->base_addr || addr >= ntb->msi->end_addr) { + dev_warn_once(&ntb->dev, + "IRQ %d: MSI Address not within the memory window (%llx, [%llx %llx])\n", + entry->irq, addr, ntb->msi->base_addr, + ntb->msi->end_addr); + return -EFAULT; + } + + msi_desc->addr_offset = addr - ntb->msi->base_addr; + msi_desc->data = entry->msg.data; + + return 0; +} + +static void ntb_msi_write_msg(struct msi_desc *entry, void *data) +{ + struct ntb_msi_devres *dr = data; + + WARN_ON(ntb_msi_set_desc(dr->ntb, entry, dr->msi_desc)); + + if (dr->ntb->msi->desc_changed) + dr->ntb->msi->desc_changed(dr->ntb->ctx); +} + +static void ntbm_msi_callback_release(struct device *dev, void *res) +{ + struct ntb_msi_devres *dr = res; + + dr->entry->write_msi_msg = NULL; + dr->entry->write_msi_msg_data = NULL; +} + +static int ntbm_msi_setup_callback(struct ntb_dev *ntb, struct msi_desc *entry, + struct ntb_msi_desc *msi_desc) +{ + struct ntb_msi_devres *dr; + + dr = devres_alloc(ntbm_msi_callback_release, + sizeof(struct ntb_msi_devres), GFP_KERNEL); + if (!dr) + return -ENOMEM; + + dr->ntb = ntb; + dr->entry = entry; + dr->msi_desc = msi_desc; + + devres_add(&ntb->dev, dr); + + dr->entry->write_msi_msg = ntb_msi_write_msg; + dr->entry->write_msi_msg_data = dr; + + return 0; +} + +/** + * ntbm_msi_request_threaded_irq() - allocate an MSI interrupt + * @ntb: NTB device context + * @handler: Function to be called when the IRQ occurs + * @thread_fn: Function to be called in a threaded interrupt context. NULL + * for clients which handle everything in @handler + * @devname: An ascii name for the claiming device, dev_name(dev) if NULL + * @dev_id: A cookie passed back to the handler function + * + * This function assigns an interrupt handler to an unused + * MSI interrupt and returns the descriptor used to trigger + * it. The descriptor can then be sent to a peer to trigger + * the interrupt. + * + * The interrupt resource is managed with devres so it will + * be automatically freed when the NTB device is torn down. + * + * If an IRQ allocated with this function needs to be freed + * separately, ntbm_free_irq() must be used. + * + * Return: IRQ number assigned on success, otherwise a negative error number. + */ +int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler, + irq_handler_t thread_fn, + const char *name, void *dev_id, + struct ntb_msi_desc *msi_desc) +{ + struct msi_desc *entry; + struct irq_desc *desc; + int ret; + + if (!ntb->msi) + return -EINVAL; + + for_each_pci_msi_entry(entry, ntb->pdev) { + desc = irq_to_desc(entry->irq); + if (desc->action) + continue; + + ret = devm_request_threaded_irq(&ntb->dev, entry->irq, handler, + thread_fn, 0, name, dev_id); + if (ret) + continue; + + if (ntb_msi_set_desc(ntb, entry, msi_desc)) { + devm_free_irq(&ntb->dev, entry->irq, dev_id); + continue; + } + + ret = ntbm_msi_setup_callback(ntb, entry, msi_desc); + if (ret) { + devm_free_irq(&ntb->dev, entry->irq, dev_id); + return ret; + } + + + return entry->irq; + } + + return -ENODEV; +} +EXPORT_SYMBOL(ntbm_msi_request_threaded_irq); + +static int ntbm_msi_callback_match(struct device *dev, void *res, void *data) +{ + struct ntb_dev *ntb = dev_ntb(dev); + struct ntb_msi_devres *dr = res; + + return dr->ntb == ntb && dr->entry == data; +} + +/** + * ntbm_msi_free_irq() - free an interrupt + * @ntb: NTB device context + * @irq: Interrupt line to free + * @dev_id: Device identity to free + * + * This function should be used to manually free IRQs allocated with + * ntbm_request_[threaded_]irq(). + */ +void ntbm_msi_free_irq(struct ntb_dev *ntb, unsigned int irq, void *dev_id) +{ + struct msi_desc *entry = irq_get_msi_desc(irq); + + entry->write_msi_msg = NULL; + entry->write_msi_msg_data = NULL; + + WARN_ON(devres_destroy(&ntb->dev, ntbm_msi_callback_release, + ntbm_msi_callback_match, entry)); + + devm_free_irq(&ntb->dev, irq, dev_id); +} +EXPORT_SYMBOL(ntbm_msi_free_irq); + +/** + * ntb_msi_peer_trigger() - Trigger an interrupt handler on a peer + * @ntb: NTB device context + * @peer: Peer index + * @desc: MSI descriptor data which triggers the interrupt + * + * This function triggers an interrupt on a peer. It requires + * the descriptor structure to have been passed from that peer + * by some other means. + * + * Return: Zero on success, otherwise a negative error number. + */ +int ntb_msi_peer_trigger(struct ntb_dev *ntb, int peer, + struct ntb_msi_desc *desc) +{ + int idx; + + if (!ntb->msi) + return -EINVAL; + + idx = desc->addr_offset / sizeof(*ntb->msi->peer_mws[peer]); + + iowrite32(desc->data, &ntb->msi->peer_mws[peer][idx]); + + return 0; +} +EXPORT_SYMBOL(ntb_msi_peer_trigger); + +/** + * ntb_msi_peer_addr() - Get the DMA address to trigger a peer's MSI interrupt + * @ntb: NTB device context + * @peer: Peer index + * @desc: MSI descriptor data which triggers the interrupt + * @msi_addr: Physical address to trigger the interrupt + * + * This function allows using DMA engines to trigger an interrupt + * (for example, trigger an interrupt to process the data after + * sending it). To trigger the interrupt, write @desc.data to the address + * returned in @msi_addr + * + * Return: Zero on success, otherwise a negative error number. + */ +int ntb_msi_peer_addr(struct ntb_dev *ntb, int peer, + struct ntb_msi_desc *desc, + phys_addr_t *msi_addr) +{ + int peer_widx = ntb_peer_mw_count(ntb) - 1 - peer; + phys_addr_t mw_phys_addr; + int ret; + + ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr, NULL); + if (ret) + return ret; + + if (msi_addr) + *msi_addr = mw_phys_addr + desc->addr_offset; + + return 0; +} +EXPORT_SYMBOL(ntb_msi_peer_addr); -- cgit v1.2.1 From a6bed7a541651f5845fba325e311f78c6a4a880a Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Thu, 23 May 2019 16:30:57 -0600 Subject: NTB: Introduce NTB MSI Test Client Introduce a tool to test NTB MSI interrupts similar to the other NTB test tools. This tool creates a debugfs directory for each NTB device with the following files: port irqX_occurrences peerX/port peerX/count peerX/trigger The 'port' file tells the user the local port number and the 'occurrences' files tell the number of local interrupts that have been received for each interrupt. For each peer, the 'port' file and the 'count' file tell you the peer's port number and number of interrupts respectively. Writing the interrupt number to the 'trigger' file triggers the interrupt handler for the peer which should increment their corresponding 'occurrences' file. The 'ready' file indicates if a peer is ready, writing to this file blocks until it is ready. The module parameter num_irqs can be used to set the number of local interrupts. By default this is 4. This is only limited by the number of unused MSI interrupts registered by the hardware (this will require support of the hardware driver) and there must be at least 2*num_irqs + 1 spads registers available. Signed-off-by: Logan Gunthorpe Cc: Dave Jiang Cc: Allen Hubbe Signed-off-by: Jon Mason --- drivers/ntb/test/Kconfig | 9 + drivers/ntb/test/Makefile | 1 + drivers/ntb/test/ntb_msi_test.c | 433 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 443 insertions(+) create mode 100644 drivers/ntb/test/ntb_msi_test.c (limited to 'drivers/ntb') diff --git a/drivers/ntb/test/Kconfig b/drivers/ntb/test/Kconfig index a5d0eda44438..a3f3e2638935 100644 --- a/drivers/ntb/test/Kconfig +++ b/drivers/ntb/test/Kconfig @@ -25,3 +25,12 @@ config NTB_PERF to and from the window without additional software interaction. If unsure, say N. + +config NTB_MSI_TEST + tristate "NTB MSI Test Client" + depends on NTB_MSI + help + This tool demonstrates the use of the NTB MSI library to + send MSI interrupts between peers. + + If unsure, say N. diff --git a/drivers/ntb/test/Makefile b/drivers/ntb/test/Makefile index 9e77e0b761c2..d2895ca995e4 100644 --- a/drivers/ntb/test/Makefile +++ b/drivers/ntb/test/Makefile @@ -1,3 +1,4 @@ obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o obj-$(CONFIG_NTB_TOOL) += ntb_tool.o obj-$(CONFIG_NTB_PERF) += ntb_perf.o +obj-$(CONFIG_NTB_MSI_TEST) += ntb_msi_test.o diff --git a/drivers/ntb/test/ntb_msi_test.c b/drivers/ntb/test/ntb_msi_test.c new file mode 100644 index 000000000000..99d826ed9c34 --- /dev/null +++ b/drivers/ntb/test/ntb_msi_test.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) + +#include +#include +#include +#include +#include +#include + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_VERSION("0.1"); +MODULE_AUTHOR("Logan Gunthorpe "); +MODULE_DESCRIPTION("Test for sending MSI interrupts over an NTB memory window"); + +static int num_irqs = 4; +module_param(num_irqs, int, 0644); +MODULE_PARM_DESC(num_irqs, "number of irqs to use"); + +struct ntb_msit_ctx { + struct ntb_dev *ntb; + struct dentry *dbgfs_dir; + struct work_struct setup_work; + + struct ntb_msit_isr_ctx { + int irq_idx; + int irq_num; + int occurrences; + struct ntb_msit_ctx *nm; + struct ntb_msi_desc desc; + } *isr_ctx; + + struct ntb_msit_peer { + struct ntb_msit_ctx *nm; + int pidx; + int num_irqs; + struct completion init_comp; + struct ntb_msi_desc *msi_desc; + } peers[]; +}; + +static struct dentry *ntb_msit_dbgfs_topdir; + +static irqreturn_t ntb_msit_isr(int irq, void *dev) +{ + struct ntb_msit_isr_ctx *isr_ctx = dev; + struct ntb_msit_ctx *nm = isr_ctx->nm; + + dev_dbg(&nm->ntb->dev, "Interrupt Occurred: %d", + isr_ctx->irq_idx); + + isr_ctx->occurrences++; + + return IRQ_HANDLED; +} + +static void ntb_msit_setup_work(struct work_struct *work) +{ + struct ntb_msit_ctx *nm = container_of(work, struct ntb_msit_ctx, + setup_work); + int irq_count = 0; + int irq; + int ret; + uintptr_t i; + + ret = ntb_msi_setup_mws(nm->ntb); + if (ret) { + dev_err(&nm->ntb->dev, "Unable to setup MSI windows: %d\n", + ret); + return; + } + + for (i = 0; i < num_irqs; i++) { + nm->isr_ctx[i].irq_idx = i; + nm->isr_ctx[i].nm = nm; + + if (!nm->isr_ctx[i].irq_num) { + irq = ntbm_msi_request_irq(nm->ntb, ntb_msit_isr, + KBUILD_MODNAME, + &nm->isr_ctx[i], + &nm->isr_ctx[i].desc); + if (irq < 0) + break; + + nm->isr_ctx[i].irq_num = irq; + } + + ret = ntb_spad_write(nm->ntb, 2 * i + 1, + nm->isr_ctx[i].desc.addr_offset); + if (ret) + break; + + ret = ntb_spad_write(nm->ntb, 2 * i + 2, + nm->isr_ctx[i].desc.data); + if (ret) + break; + + irq_count++; + } + + ntb_spad_write(nm->ntb, 0, irq_count); + ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb))); +} + +static void ntb_msit_desc_changed(void *ctx) +{ + struct ntb_msit_ctx *nm = ctx; + int i; + + dev_dbg(&nm->ntb->dev, "MSI Descriptors Changed\n"); + + for (i = 0; i < num_irqs; i++) { + ntb_spad_write(nm->ntb, 2 * i + 1, + nm->isr_ctx[i].desc.addr_offset); + ntb_spad_write(nm->ntb, 2 * i + 2, + nm->isr_ctx[i].desc.data); + } + + ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb))); +} + +static void ntb_msit_link_event(void *ctx) +{ + struct ntb_msit_ctx *nm = ctx; + + if (!ntb_link_is_up(nm->ntb, NULL, NULL)) + return; + + schedule_work(&nm->setup_work); +} + +static void ntb_msit_copy_peer_desc(struct ntb_msit_ctx *nm, int peer) +{ + int i; + struct ntb_msi_desc *desc = nm->peers[peer].msi_desc; + int irq_count = nm->peers[peer].num_irqs; + + for (i = 0; i < irq_count; i++) { + desc[i].addr_offset = ntb_peer_spad_read(nm->ntb, peer, + 2 * i + 1); + desc[i].data = ntb_peer_spad_read(nm->ntb, peer, 2 * i + 2); + } + + dev_info(&nm->ntb->dev, "Found %d interrupts on peer %d\n", + irq_count, peer); + + complete_all(&nm->peers[peer].init_comp); +} + +static void ntb_msit_db_event(void *ctx, int vec) +{ + struct ntb_msit_ctx *nm = ctx; + struct ntb_msi_desc *desc; + u64 peer_mask = ntb_db_read(nm->ntb); + u32 irq_count; + int peer; + + ntb_db_clear(nm->ntb, peer_mask); + + for (peer = 0; peer < sizeof(peer_mask) * 8; peer++) { + if (!(peer_mask & BIT(peer))) + continue; + + irq_count = ntb_peer_spad_read(nm->ntb, peer, 0); + if (irq_count == -1) + continue; + + desc = kcalloc(irq_count, sizeof(*desc), GFP_ATOMIC); + if (!desc) + continue; + + kfree(nm->peers[peer].msi_desc); + nm->peers[peer].msi_desc = desc; + nm->peers[peer].num_irqs = irq_count; + + ntb_msit_copy_peer_desc(nm, peer); + } +} + +static const struct ntb_ctx_ops ntb_msit_ops = { + .link_event = ntb_msit_link_event, + .db_event = ntb_msit_db_event, +}; + +static int ntb_msit_dbgfs_trigger(void *data, u64 idx) +{ + struct ntb_msit_peer *peer = data; + + if (idx >= peer->num_irqs) + return -EINVAL; + + dev_dbg(&peer->nm->ntb->dev, "trigger irq %llu on peer %u\n", + idx, peer->pidx); + + return ntb_msi_peer_trigger(peer->nm->ntb, peer->pidx, + &peer->msi_desc[idx]); +} + +DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_trigger_fops, NULL, + ntb_msit_dbgfs_trigger, "%llu\n"); + +static int ntb_msit_dbgfs_port_get(void *data, u64 *port) +{ + struct ntb_msit_peer *peer = data; + + *port = ntb_peer_port_number(peer->nm->ntb, peer->pidx); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_port_fops, ntb_msit_dbgfs_port_get, + NULL, "%llu\n"); + +static int ntb_msit_dbgfs_count_get(void *data, u64 *count) +{ + struct ntb_msit_peer *peer = data; + + *count = peer->num_irqs; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_count_fops, ntb_msit_dbgfs_count_get, + NULL, "%llu\n"); + +static int ntb_msit_dbgfs_ready_get(void *data, u64 *ready) +{ + struct ntb_msit_peer *peer = data; + + *ready = try_wait_for_completion(&peer->init_comp); + + return 0; +} + +static int ntb_msit_dbgfs_ready_set(void *data, u64 ready) +{ + struct ntb_msit_peer *peer = data; + + return wait_for_completion_interruptible(&peer->init_comp); +} + +DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_ready_fops, ntb_msit_dbgfs_ready_get, + ntb_msit_dbgfs_ready_set, "%llu\n"); + +static int ntb_msit_dbgfs_occurrences_get(void *data, u64 *occurrences) +{ + struct ntb_msit_isr_ctx *isr_ctx = data; + + *occurrences = isr_ctx->occurrences; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_occurrences_fops, + ntb_msit_dbgfs_occurrences_get, + NULL, "%llu\n"); + +static int ntb_msit_dbgfs_local_port_get(void *data, u64 *port) +{ + struct ntb_msit_ctx *nm = data; + + *port = ntb_port_number(nm->ntb); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_local_port_fops, + ntb_msit_dbgfs_local_port_get, + NULL, "%llu\n"); + +static void ntb_msit_create_dbgfs(struct ntb_msit_ctx *nm) +{ + struct pci_dev *pdev = nm->ntb->pdev; + char buf[32]; + int i; + struct dentry *peer_dir; + + nm->dbgfs_dir = debugfs_create_dir(pci_name(pdev), + ntb_msit_dbgfs_topdir); + debugfs_create_file("port", 0400, nm->dbgfs_dir, nm, + &ntb_msit_local_port_fops); + + for (i = 0; i < ntb_peer_port_count(nm->ntb); i++) { + nm->peers[i].pidx = i; + nm->peers[i].nm = nm; + init_completion(&nm->peers[i].init_comp); + + snprintf(buf, sizeof(buf), "peer%d", i); + peer_dir = debugfs_create_dir(buf, nm->dbgfs_dir); + + debugfs_create_file_unsafe("trigger", 0200, peer_dir, + &nm->peers[i], + &ntb_msit_trigger_fops); + + debugfs_create_file_unsafe("port", 0400, peer_dir, + &nm->peers[i], &ntb_msit_port_fops); + + debugfs_create_file_unsafe("count", 0400, peer_dir, + &nm->peers[i], + &ntb_msit_count_fops); + + debugfs_create_file_unsafe("ready", 0600, peer_dir, + &nm->peers[i], + &ntb_msit_ready_fops); + } + + for (i = 0; i < num_irqs; i++) { + snprintf(buf, sizeof(buf), "irq%d_occurrences", i); + debugfs_create_file_unsafe(buf, 0400, nm->dbgfs_dir, + &nm->isr_ctx[i], + &ntb_msit_occurrences_fops); + } +} + +static void ntb_msit_remove_dbgfs(struct ntb_msit_ctx *nm) +{ + debugfs_remove_recursive(nm->dbgfs_dir); +} + +static int ntb_msit_probe(struct ntb_client *client, struct ntb_dev *ntb) +{ + struct ntb_msit_ctx *nm; + size_t struct_size; + int peers; + int ret; + + peers = ntb_peer_port_count(ntb); + if (peers <= 0) + return -EINVAL; + + if (ntb_spad_is_unsafe(ntb) || ntb_spad_count(ntb) < 2 * num_irqs + 1) { + dev_err(&ntb->dev, "NTB MSI test requires at least %d spads for %d irqs\n", + 2 * num_irqs + 1, num_irqs); + return -EFAULT; + } + + ret = ntb_spad_write(ntb, 0, -1); + if (ret) { + dev_err(&ntb->dev, "Unable to write spads: %d\n", ret); + return ret; + } + + ret = ntb_db_clear_mask(ntb, GENMASK(peers - 1, 0)); + if (ret) { + dev_err(&ntb->dev, "Unable to clear doorbell mask: %d\n", ret); + return ret; + } + + ret = ntb_msi_init(ntb, ntb_msit_desc_changed); + if (ret) { + dev_err(&ntb->dev, "Unable to initialize MSI library: %d\n", + ret); + return ret; + } + + struct_size = sizeof(*nm) + sizeof(*nm->peers) * peers; + + nm = devm_kzalloc(&ntb->dev, struct_size, GFP_KERNEL); + if (!nm) + return -ENOMEM; + + nm->isr_ctx = devm_kcalloc(&ntb->dev, num_irqs, sizeof(*nm->isr_ctx), + GFP_KERNEL); + if (!nm->isr_ctx) + return -ENOMEM; + + INIT_WORK(&nm->setup_work, ntb_msit_setup_work); + nm->ntb = ntb; + + ntb_msit_create_dbgfs(nm); + + ret = ntb_set_ctx(ntb, nm, &ntb_msit_ops); + if (ret) + goto remove_dbgfs; + + if (!nm->isr_ctx) + goto remove_dbgfs; + + ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + + return 0; + +remove_dbgfs: + ntb_msit_remove_dbgfs(nm); + devm_kfree(&ntb->dev, nm->isr_ctx); + devm_kfree(&ntb->dev, nm); + return ret; +} + +static void ntb_msit_remove(struct ntb_client *client, struct ntb_dev *ntb) +{ + struct ntb_msit_ctx *nm = ntb->ctx; + int i; + + ntb_link_disable(ntb); + ntb_db_set_mask(ntb, ntb_db_valid_mask(ntb)); + ntb_msi_clear_mws(ntb); + + for (i = 0; i < ntb_peer_port_count(ntb); i++) + kfree(nm->peers[i].msi_desc); + + ntb_clear_ctx(ntb); + ntb_msit_remove_dbgfs(nm); +} + +static struct ntb_client ntb_msit_client = { + .ops = { + .probe = ntb_msit_probe, + .remove = ntb_msit_remove + } +}; + +static int __init ntb_msit_init(void) +{ + int ret; + + if (debugfs_initialized()) + ntb_msit_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, + NULL); + + ret = ntb_register_client(&ntb_msit_client); + if (ret) + debugfs_remove_recursive(ntb_msit_dbgfs_topdir); + + return ret; +} +module_init(ntb_msit_init); + +static void __exit ntb_msit_exit(void) +{ + ntb_unregister_client(&ntb_msit_client); + debugfs_remove_recursive(ntb_msit_dbgfs_topdir); +} +module_exit(ntb_msit_exit); -- cgit v1.2.1 From 2b0569b3b7e6eac7a0c5aef741aeebe5675e1640 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Thu, 23 May 2019 16:30:59 -0600 Subject: NTB: Add MSI interrupt support to ntb_transport Introduce the module parameter 'use_msi' which, when set, uses MSI interrupts instead of doorbells for each queue pair (QP). The parameter is only available if NTB MSI support is configured into the kernel. We also require there to be more than one memory window (MW) so that an extra one is available to forward the APIC region. To use MSIs, we request one interrupt per QP and forward the MSI address and data to the peer using scratch pad registers (SPADS) above the MW SPADS. (If there are not enough SPADS the MSI interrupt will not be used.) Once registered, we simply use ntb_msi_peer_trigger and the receiving ISR simply queues up the rxc_db_work for the queue. This addition can significantly improve performance of ntb_transport. In a simple, untuned, apples-to-apples comparision using ntb_netdev and iperf with switchtec hardware, I see 3.88Gb/s without MSI interrupts and 14.1Gb/s wit MSI, which is a more than 3x improvement. Signed-off-by: Logan Gunthorpe Cc: Dave Jiang Cc: Allen Hubbe Signed-off-by: Jon Mason --- drivers/ntb/ntb_transport.c | 169 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 1 deletion(-) (limited to 'drivers/ntb') diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 99033228dccb..40c90ca10729 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -93,6 +93,12 @@ static bool use_dma; module_param(use_dma, bool, 0644); MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); +static bool use_msi; +#ifdef CONFIG_NTB_MSI +module_param(use_msi, bool, 0644); +MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells"); +#endif + static struct dentry *nt_debugfs_dir; /* Only two-ports NTB devices are supported */ @@ -188,6 +194,11 @@ struct ntb_transport_qp { u64 tx_err_no_buf; u64 tx_memcpy; u64 tx_async; + + bool use_msi; + int msi_irq; + struct ntb_msi_desc msi_desc; + struct ntb_msi_desc peer_msi_desc; }; struct ntb_transport_mw { @@ -221,6 +232,10 @@ struct ntb_transport_ctx { u64 qp_bitmap; u64 qp_bitmap_free; + bool use_msi; + unsigned int msi_spad_offset; + u64 msi_db_mask; + bool link_is_up; struct delayed_work link_work; struct work_struct link_cleanup; @@ -667,6 +682,114 @@ static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, return 0; } +static irqreturn_t ntb_transport_isr(int irq, void *dev) +{ + struct ntb_transport_qp *qp = dev; + + tasklet_schedule(&qp->rxc_db_work); + + return IRQ_HANDLED; +} + +static void ntb_transport_setup_qp_peer_msi(struct ntb_transport_ctx *nt, + unsigned int qp_num) +{ + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; + int spad = qp_num * 2 + nt->msi_spad_offset; + + if (!nt->use_msi) + return; + + if (spad >= ntb_spad_count(nt->ndev)) + return; + + qp->peer_msi_desc.addr_offset = + ntb_peer_spad_read(qp->ndev, PIDX, spad); + qp->peer_msi_desc.data = + ntb_peer_spad_read(qp->ndev, PIDX, spad + 1); + + dev_dbg(&qp->ndev->pdev->dev, "QP%d Peer MSI addr=%x data=%x\n", + qp_num, qp->peer_msi_desc.addr_offset, qp->peer_msi_desc.data); + + if (qp->peer_msi_desc.addr_offset) { + qp->use_msi = true; + dev_info(&qp->ndev->pdev->dev, + "Using MSI interrupts for QP%d\n", qp_num); + } +} + +static void ntb_transport_setup_qp_msi(struct ntb_transport_ctx *nt, + unsigned int qp_num) +{ + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; + int spad = qp_num * 2 + nt->msi_spad_offset; + int rc; + + if (!nt->use_msi) + return; + + if (spad >= ntb_spad_count(nt->ndev)) { + dev_warn_once(&qp->ndev->pdev->dev, + "Not enough SPADS to use MSI interrupts\n"); + return; + } + + ntb_spad_write(qp->ndev, spad, 0); + ntb_spad_write(qp->ndev, spad + 1, 0); + + if (!qp->msi_irq) { + qp->msi_irq = ntbm_msi_request_irq(qp->ndev, ntb_transport_isr, + KBUILD_MODNAME, qp, + &qp->msi_desc); + if (qp->msi_irq < 0) { + dev_warn(&qp->ndev->pdev->dev, + "Unable to allocate MSI interrupt for qp%d\n", + qp_num); + return; + } + } + + rc = ntb_spad_write(qp->ndev, spad, qp->msi_desc.addr_offset); + if (rc) + goto err_free_interrupt; + + rc = ntb_spad_write(qp->ndev, spad + 1, qp->msi_desc.data); + if (rc) + goto err_free_interrupt; + + dev_dbg(&qp->ndev->pdev->dev, "QP%d MSI %d addr=%x data=%x\n", + qp_num, qp->msi_irq, qp->msi_desc.addr_offset, + qp->msi_desc.data); + + return; + +err_free_interrupt: + devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp); +} + +static void ntb_transport_msi_peer_desc_changed(struct ntb_transport_ctx *nt) +{ + int i; + + dev_dbg(&nt->ndev->pdev->dev, "Peer MSI descriptors changed"); + + for (i = 0; i < nt->qp_count; i++) + ntb_transport_setup_qp_peer_msi(nt, i); +} + +static void ntb_transport_msi_desc_changed(void *data) +{ + struct ntb_transport_ctx *nt = data; + int i; + + dev_dbg(&nt->ndev->pdev->dev, "MSI descriptors changed"); + + for (i = 0; i < nt->qp_count; i++) + ntb_transport_setup_qp_msi(nt, i); + + ntb_peer_db_set(nt->ndev, nt->msi_db_mask); +} + static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) { struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; @@ -905,6 +1028,20 @@ static void ntb_transport_link_work(struct work_struct *work) int rc = 0, i, spad; /* send the local info, in the opposite order of the way we read it */ + + if (nt->use_msi) { + rc = ntb_msi_setup_mws(ndev); + if (rc) { + dev_warn(&pdev->dev, + "Failed to register MSI memory window: %d\n", + rc); + nt->use_msi = false; + } + } + + for (i = 0; i < nt->qp_count; i++) + ntb_transport_setup_qp_msi(nt, i); + for (i = 0; i < nt->mw_count; i++) { size = nt->mw_vec[i].phys_size; @@ -962,6 +1099,7 @@ static void ntb_transport_link_work(struct work_struct *work) struct ntb_transport_qp *qp = &nt->qp_vec[i]; ntb_transport_setup_qp_mw(nt, i); + ntb_transport_setup_qp_peer_msi(nt, i); if (qp->client_ready) schedule_delayed_work(&qp->link_work, 0); @@ -1135,6 +1273,19 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) return -ENOMEM; nt->ndev = ndev; + + /* + * If we are using MSI, and have at least one extra memory window, + * we will reserve the last MW for the MSI window. + */ + if (use_msi && mw_count > 1) { + rc = ntb_msi_init(ndev, ntb_transport_msi_desc_changed); + if (!rc) { + mw_count -= 1; + nt->use_msi = true; + } + } + spad_count = ntb_spad_count(ndev); /* Limit the MW's based on the availability of scratchpads */ @@ -1148,6 +1299,8 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2; nt->mw_count = min(mw_count, max_mw_count_for_spads); + nt->msi_spad_offset = nt->mw_count * 2 + MW0_SZ_HIGH; + nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec), GFP_KERNEL, node); if (!nt->mw_vec) { @@ -1178,6 +1331,12 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) qp_bitmap = ntb_db_valid_mask(ndev); qp_count = ilog2(qp_bitmap); + if (nt->use_msi) { + qp_count -= 1; + nt->msi_db_mask = 1 << qp_count; + ntb_db_clear_mask(ndev, nt->msi_db_mask); + } + if (max_num_clients && max_num_clients < qp_count) qp_count = max_num_clients; else if (nt->mw_count < qp_count) @@ -1601,7 +1760,10 @@ static void ntb_tx_copy_callback(void *data, iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); - ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); + if (qp->use_msi) + ntb_msi_peer_trigger(qp->ndev, PIDX, &qp->peer_msi_desc); + else + ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); /* The entry length can only be zero if the packet is intended to be a * "link down" or similar. Since no payload is being sent in these @@ -2269,6 +2431,11 @@ static void ntb_transport_doorbell_callback(void *data, int vector) u64 db_bits; unsigned int qp_num; + if (ntb_db_read(nt->ndev) & nt->msi_db_mask) { + ntb_transport_msi_peer_desc_changed(nt); + ntb_db_clear(nt->ndev, nt->msi_db_mask); + } + db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & ntb_db_vector_mask(nt->ndev, vector)); -- cgit v1.2.1