diff options
222 files changed, 8671 insertions, 1701 deletions
diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt index e74515ecaa9c..8313a636dd53 100644 --- a/Documentation/networking/ip-sysctl.txt +++ b/Documentation/networking/ip-sysctl.txt @@ -1882,6 +1882,11 @@ ratelimit - INTEGER otherwise the minimal space between responses in milliseconds. Default: 1000 +echo_ignore_all - BOOLEAN + If set non-zero, then the kernel will ignore all ICMP ECHO + requests sent to it over the IPv6 protocol. + Default: 0 + xfrm6_gc_thresh - INTEGER The threshold at which we will start garbage collecting for IPv6 destination cache entries. At twice this value the system will diff --git a/MAINTAINERS b/MAINTAINERS index 9777a1dd6ff6..4c05e18ced50 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5930,7 +5930,7 @@ F: Documentation/dev-tools/gcov.rst GDB KERNEL DEBUGGING HELPER SCRIPTS M: Jan Kiszka <jan.kiszka@siemens.com> -M: Kieran Bingham <kieran@bingham.xyz> +M: Kieran Bingham <kbingham@kernel.org> S: Supported F: scripts/gdb/ diff --git a/arch/mips/configs/bcm47xx_defconfig b/arch/mips/configs/bcm47xx_defconfig index fad8e964f14c..ba800a892384 100644 --- a/arch/mips/configs/bcm47xx_defconfig +++ b/arch/mips/configs/bcm47xx_defconfig @@ -66,7 +66,6 @@ CONFIG_HW_RANDOM=y CONFIG_GPIO_SYSFS=y CONFIG_WATCHDOG=y CONFIG_BCM47XX_WDT=y -CONFIG_SSB_DEBUG=y CONFIG_SSB_DRIVER_GIGE=y CONFIG_BCMA_DRIVER_GMAC_CMN=y CONFIG_USB=y diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 46f656b8fc23..e7705dde953f 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -11,7 +11,6 @@ config PARISC select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_WANTS_UBSAN_NO_NULL select ARCH_SUPPORTS_MEMORY_FAILURE select RTC_CLASS select RTC_DRV_GENERIC diff --git a/arch/powerpc/configs/wii_defconfig b/arch/powerpc/configs/wii_defconfig index 10940533da71..f5c366b02828 100644 --- a/arch/powerpc/configs/wii_defconfig +++ b/arch/powerpc/configs/wii_defconfig @@ -78,7 +78,6 @@ CONFIG_GPIO_HLWD=y CONFIG_POWER_RESET=y CONFIG_POWER_RESET_GPIO=y # CONFIG_HWMON is not set -CONFIG_SSB_DEBUG=y CONFIG_FB=y # CONFIG_VGA_CONSOLE is not set CONFIG_FRAMEBUFFER_CONSOLE=y diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 8a1863d9ed53..4fe5b2affa23 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -106,7 +106,6 @@ config S390 select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF select ARCH_WANTS_DYNAMIC_TASK_STRUCT - select ARCH_WANTS_UBSAN_NO_NULL select ARCH_WANT_IPC_PARSE_VERSION select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS2 diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 7436b2d27fa3..a390c6d4f72d 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -298,7 +298,8 @@ static void reset_bdev(struct zram *zram) zram->backing_dev = NULL; zram->old_block_size = 0; zram->bdev = NULL; - + zram->disk->queue->backing_dev_info->capabilities |= + BDI_CAP_SYNCHRONOUS_IO; kvfree(zram->bitmap); zram->bitmap = NULL; } @@ -400,6 +401,18 @@ static ssize_t backing_dev_store(struct device *dev, zram->backing_dev = backing_dev; zram->bitmap = bitmap; zram->nr_pages = nr_pages; + /* + * With writeback feature, zram does asynchronous IO so it's no longer + * synchronous device so let's remove synchronous io flag. Othewise, + * upper layer(e.g., swap) could wait IO completion rather than + * (submit and return), which will cause system sluggish. + * Furthermore, when the IO function returns(e.g., swap_readpage), + * upper layer expects IO was done so it could deallocate the page + * freely but in fact, IO is going on so finally could cause + * use-after-free when the IO is really done. + */ + zram->disk->queue->backing_dev_info->capabilities &= + ~BDI_CAP_SYNCHRONOUS_IO; up_write(&zram->init_lock); pr_info("setup backing device %s\n", file_name); diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 83326986c113..7c7198553699 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -146,7 +146,7 @@ static void virtcrypto_clean_affinity(struct virtio_crypto *vi, long hcpu) if (vi->affinity_hint_set) { for (i = 0; i < vi->max_data_queues; i++) - virtqueue_set_affinity(vi->data_vq[i].vq, -1); + virtqueue_set_affinity(vi->data_vq[i].vq, NULL); vi->affinity_hint_set = false; } @@ -173,7 +173,7 @@ static void virtcrypto_set_affinity(struct virtio_crypto *vcrypto) * */ for_each_online_cpu(cpu) { - virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpu); + virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpumask_of(cpu)); if (++i >= vcrypto->max_data_queues) break; } diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c index 1f41a4f89c08..8a873975cf12 100644 --- a/drivers/i2c/busses/i2c-xlp9xx.c +++ b/drivers/i2c/busses/i2c-xlp9xx.c @@ -191,28 +191,43 @@ static void xlp9xx_i2c_drain_rx_fifo(struct xlp9xx_i2c_dev *priv) if (priv->len_recv) { /* read length byte */ rlen = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_MRXFIFO); + + /* + * We expect at least 2 interrupts for I2C_M_RECV_LEN + * transactions. The length is updated during the first + * interrupt, and the buffer contents are only copied + * during subsequent interrupts. If in case the interrupts + * get merged we would complete the transaction without + * copying out the bytes from RX fifo. To avoid this now we + * drain the fifo as and when data is available. + * We drained the rlen byte already, decrement total length + * by one. + */ + + len--; if (rlen > I2C_SMBUS_BLOCK_MAX || rlen == 0) { rlen = 0; /*abort transfer */ priv->msg_buf_remaining = 0; priv->msg_len = 0; - } else { - *buf++ = rlen; - if (priv->client_pec) - ++rlen; /* account for error check byte */ - /* update remaining bytes and message length */ - priv->msg_buf_remaining = rlen; - priv->msg_len = rlen + 1; + xlp9xx_i2c_update_rlen(priv); + return; } + + *buf++ = rlen; + if (priv->client_pec) + ++rlen; /* account for error check byte */ + /* update remaining bytes and message length */ + priv->msg_buf_remaining = rlen; + priv->msg_len = rlen + 1; xlp9xx_i2c_update_rlen(priv); priv->len_recv = false; - } else { - len = min(priv->msg_buf_remaining, len); - for (i = 0; i < len; i++, buf++) - *buf = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_MRXFIFO); - - priv->msg_buf_remaining -= len; } + len = min(priv->msg_buf_remaining, len); + for (i = 0; i < len; i++, buf++) + *buf = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_MRXFIFO); + + priv->msg_buf_remaining -= len; priv->msg_buf = buf; if (priv->msg_buf_remaining) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index b6dbc3f6d309..9c929cd90b86 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -2823,7 +2823,7 @@ bnxt_fill_coredump_seg_hdr(struct bnxt *bp, int status, u32 duration, u32 instance) { memset(seg_hdr, 0, sizeof(*seg_hdr)); - strcpy(seg_hdr->signature, "sEgM"); + memcpy(seg_hdr->signature, "sEgM", 4); if (seg_rec) { seg_hdr->component_id = (__force __le32)seg_rec->component_id; seg_hdr->segment_id = (__force __le32)seg_rec->segment_id; @@ -2855,7 +2855,7 @@ bnxt_fill_coredump_record(struct bnxt *bp, struct bnxt_coredump_record *record, time64_to_tm(start, 0, &tm); memset(record, 0, sizeof(*record)); - strcpy(record->signature, "cOrE"); + memcpy(record->signature, "cOrE", 4); record->flags = 0; record->low_version = 0; record->high_version = 1; @@ -2876,7 +2876,7 @@ bnxt_fill_coredump_record(struct bnxt *bp, struct bnxt_coredump_record *record, record->os_ver_major = cpu_to_le32(os_ver_major); record->os_ver_minor = cpu_to_le32(os_ver_minor); - strcpy(record->os_name, utsname()->sysname); + strlcpy(record->os_name, utsname()->sysname, 32); time64_to_tm(end, 0, &tm); record->end_year = cpu_to_le16(tm.tm_year + 1900); record->end_month = cpu_to_le16(tm.tm_mon + 1); diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c index 06f7449c569d..8e05afd5e39c 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c @@ -389,18 +389,14 @@ static int lio_set_link_ksettings(struct net_device *netdev, struct lio *lio = GET_LIO(netdev); struct oct_link_info *linfo; struct octeon_device *oct; - u32 is25G = 0; oct = lio->oct_dev; linfo = &lio->linfo; - if (oct->subsystem_id == OCTEON_CN2350_25GB_SUBSYS_ID || - oct->subsystem_id == OCTEON_CN2360_25GB_SUBSYS_ID) { - is25G = 1; - } else { + if (!(oct->subsystem_id == OCTEON_CN2350_25GB_SUBSYS_ID || + oct->subsystem_id == OCTEON_CN2360_25GB_SUBSYS_ID)) return -EOPNOTSUPP; - } if (oct->no_speed_setting) { dev_err(&oct->pci_dev->dev, "%s: Changing speed is not supported\n", @@ -857,7 +853,14 @@ static int lio_set_phys_id(struct net_device *netdev, { struct lio *lio = GET_LIO(netdev); struct octeon_device *oct = lio->oct_dev; + struct oct_link_info *linfo; int value, ret; + u32 cur_ver; + + linfo = &lio->linfo; + cur_ver = OCT_FW_VER(oct->fw_info.ver.maj, + oct->fw_info.ver.min, + oct->fw_info.ver.rev); switch (state) { case ETHTOOL_ID_ACTIVE: @@ -896,16 +899,22 @@ static int lio_set_phys_id(struct net_device *netdev, return ret; } else if (oct->chip_id == OCTEON_CN23XX_PF_VID) { octnet_id_active(netdev, LED_IDENTIFICATION_ON); - - /* returns 0 since updates are asynchronous */ - return 0; + if (linfo->link.s.phy_type == LIO_PHY_PORT_TP && + cur_ver > OCT_FW_VER(1, 7, 2)) + return 2; + else + return 0; } else { return -EINVAL; } break; case ETHTOOL_ID_ON: - if (oct->chip_id == OCTEON_CN66XX) + if (oct->chip_id == OCTEON_CN23XX_PF_VID && + linfo->link.s.phy_type == LIO_PHY_PORT_TP && + cur_ver > OCT_FW_VER(1, 7, 2)) + octnet_id_active(netdev, LED_IDENTIFICATION_ON); + else if (oct->chip_id == OCTEON_CN66XX) octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG, VITESSE_PHY_GPIO_HIGH); else @@ -914,7 +923,11 @@ static int lio_set_phys_id(struct net_device *netdev, break; case ETHTOOL_ID_OFF: - if (oct->chip_id == OCTEON_CN66XX) + if (oct->chip_id == OCTEON_CN23XX_PF_VID && + linfo->link.s.phy_type == LIO_PHY_PORT_TP && + cur_ver > OCT_FW_VER(1, 7, 2)) + octnet_id_active(netdev, LED_IDENTIFICATION_OFF); + else if (oct->chip_id == OCTEON_CN66XX) octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG, VITESSE_PHY_GPIO_LOW); else diff --git a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h index 690424b6781a..7407fcd338e9 100644 --- a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h +++ b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h @@ -907,6 +907,7 @@ static inline int opcode_slow_path(union octeon_rh *rh) #define VITESSE_PHY_GPIO_LOW 0x3 #define LED_IDENTIFICATION_ON 0x1 #define LED_IDENTIFICATION_OFF 0x0 +#define LIO23XX_COPPERHEAD_LED_GPIO 0x2 struct oct_mdio_cmd { u64 op; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index 69590cff2a2d..961e3087d1d3 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -5144,17 +5144,9 @@ static void print_port_info(const struct net_device *dev) { char buf[80]; char *bufp = buf; - const char *spd = ""; const struct port_info *pi = netdev_priv(dev); const struct adapter *adap = pi->adapter; - if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB) - spd = " 2.5 GT/s"; - else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB) - spd = " 5 GT/s"; - else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB) - spd = " 8 GT/s"; - if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M) bufp += sprintf(bufp, "100M/"); if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G) diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 55c2a56c5dae..bc80a678abc3 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -4107,13 +4107,18 @@ static int mvneta_config_rss(struct mvneta_port *pp) on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); - /* We have to synchronise on the napi of each CPU */ - for_each_online_cpu(cpu) { - struct mvneta_pcpu_port *pcpu_port = - per_cpu_ptr(pp->ports, cpu); + if (!pp->neta_armada3700) { + /* We have to synchronise on the napi of each CPU */ + for_each_online_cpu(cpu) { + struct mvneta_pcpu_port *pcpu_port = + per_cpu_ptr(pp->ports, cpu); - napi_synchronize(&pcpu_port->napi); - napi_disable(&pcpu_port->napi); + napi_synchronize(&pcpu_port->napi); + napi_disable(&pcpu_port->napi); + } + } else { + napi_synchronize(&pp->napi); + napi_disable(&pp->napi); } pp->rxq_def = pp->indir[0]; @@ -4130,12 +4135,16 @@ static int mvneta_config_rss(struct mvneta_port *pp) mvneta_percpu_elect(pp); spin_unlock(&pp->lock); - /* We have to synchronise on the napi of each CPU */ - for_each_online_cpu(cpu) { - struct mvneta_pcpu_port *pcpu_port = - per_cpu_ptr(pp->ports, cpu); + if (!pp->neta_armada3700) { + /* We have to synchronise on the napi of each CPU */ + for_each_online_cpu(cpu) { + struct mvneta_pcpu_port *pcpu_port = + per_cpu_ptr(pp->ports, cpu); - napi_enable(&pcpu_port->napi); + napi_enable(&pcpu_port->napi); + } + } else { + napi_enable(&pp->napi); } netif_tx_start_all_queues(pp->dev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index b5d5e57f1583..81533d7f395c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -726,15 +726,6 @@ static struct mlxsw_driver *mlxsw_core_driver_get(const char *kind) return mlxsw_driver; } -static void mlxsw_core_driver_put(const char *kind) -{ - struct mlxsw_driver *mlxsw_driver; - - spin_lock(&mlxsw_core_driver_list_lock); - mlxsw_driver = __driver_find(kind); - spin_unlock(&mlxsw_core_driver_list_lock); -} - static int mlxsw_devlink_port_split(struct devlink *devlink, unsigned int port_index, unsigned int count, @@ -1082,7 +1073,6 @@ err_bus_init: if (!reload) devlink_free(devlink); err_devlink_alloc: - mlxsw_core_driver_put(device_kind); return err; } EXPORT_SYMBOL(mlxsw_core_bus_device_register); @@ -1090,7 +1080,6 @@ EXPORT_SYMBOL(mlxsw_core_bus_device_register); void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, bool reload) { - const char *device_kind = mlxsw_core->bus_info->device_kind; struct devlink *devlink = priv_to_devlink(mlxsw_core); if (mlxsw_core->reload_fail) @@ -1111,7 +1100,6 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, return; reload_fail: devlink_free(devlink); - mlxsw_core_driver_put(device_kind); } EXPORT_SYMBOL(mlxsw_core_bus_device_unregister); diff --git a/drivers/net/ethernet/microchip/Makefile b/drivers/net/ethernet/microchip/Makefile index 43f47cb45fe2..538926d2b43f 100644 --- a/drivers/net/ethernet/microchip/Makefile +++ b/drivers/net/ethernet/microchip/Makefile @@ -6,4 +6,4 @@ obj-$(CONFIG_ENC28J60) += enc28j60.o obj-$(CONFIG_ENCX24J600) += encx24j600.o encx24j600-regmap.o obj-$(CONFIG_LAN743X) += lan743x.o -lan743x-objs := lan743x_main.o lan743x_ethtool.o +lan743x-objs := lan743x_main.o lan743x_ethtool.o lan743x_ptp.o diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index c25b3e97ae26..07c1eb63415a 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -4,6 +4,7 @@ #include <linux/netdevice.h> #include "lan743x_main.h" #include "lan743x_ethtool.h" +#include <linux/net_tstamp.h> #include <linux/pci.h> #include <linux/phy.h> @@ -542,6 +543,31 @@ static int lan743x_ethtool_set_rxfh(struct net_device *netdev, return 0; } +static int lan743x_ethtool_get_ts_info(struct net_device *netdev, + struct ethtool_ts_info *ts_info) +{ + struct lan743x_adapter *adapter = netdev_priv(netdev); + + ts_info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + if (adapter->ptp.ptp_clock) + ts_info->phc_index = ptp_clock_index(adapter->ptp.ptp_clock); + else + ts_info->phc_index = -1; + + ts_info->tx_types = BIT(HWTSTAMP_TX_OFF) | + BIT(HWTSTAMP_TX_ON) | + BIT(HWTSTAMP_TX_ONESTEP_SYNC); + ts_info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_ALL); + return 0; +} + static int lan743x_ethtool_get_eee(struct net_device *netdev, struct ethtool_eee *eee) { @@ -685,6 +711,7 @@ const struct ethtool_ops lan743x_ethtool_ops = { .get_rxfh_indir_size = lan743x_ethtool_get_rxfh_indir_size, .get_rxfh = lan743x_ethtool_get_rxfh, .set_rxfh = lan743x_ethtool_set_rxfh, + .get_ts_info = lan743x_ethtool_get_ts_info, .get_eee = lan743x_ethtool_get_eee, .set_eee = lan743x_ethtool_set_eee, .get_link_ksettings = phy_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index bb323f269839..e7dce79ff2c9 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -267,6 +267,10 @@ static void lan743x_intr_shared_isr(void *context, u32 int_sts, u32 flags) lan743x_intr_software_isr(adapter); int_sts &= ~INT_BIT_SW_GP_; } + if (int_sts & INT_BIT_1588_) { + lan743x_ptp_isr(adapter); + int_sts &= ~INT_BIT_1588_; + } } if (int_sts) lan743x_csr_write(adapter, INT_EN_CLR, int_sts); @@ -976,6 +980,7 @@ static void lan743x_phy_link_status_change(struct net_device *netdev) ksettings.base.duplex, local_advertisement, remote_advertisement); + lan743x_ptp_update_latency(adapter, ksettings.base.speed); } } @@ -1226,6 +1231,7 @@ static void lan743x_tx_release_desc(struct lan743x_tx *tx, struct lan743x_tx_buffer_info *buffer_info = NULL; struct lan743x_tx_descriptor *descriptor = NULL; u32 descriptor_type = 0; + bool ignore_sync; descriptor = &tx->ring_cpu_ptr[descriptor_index]; buffer_info = &tx->buffer_info[descriptor_index]; @@ -1256,11 +1262,27 @@ clean_up_data_descriptor: buffer_info->dma_ptr = 0; buffer_info->buffer_length = 0; } - if (buffer_info->skb) { + if (!buffer_info->skb) + goto clear_active; + + if (!(buffer_info->flags & TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED)) { dev_kfree_skb(buffer_info->skb); - buffer_info->skb = NULL; + goto clear_skb; + } + + if (cleanup) { + lan743x_ptp_unrequest_tx_timestamp(tx->adapter); + dev_kfree_skb(buffer_info->skb); + } else { + ignore_sync = (buffer_info->flags & + TX_BUFFER_INFO_FLAG_IGNORE_SYNC) != 0; + lan743x_ptp_tx_timestamp_skb(tx->adapter, + buffer_info->skb, ignore_sync); } +clear_skb: + buffer_info->skb = NULL; + clear_active: buffer_info->flags &= ~TX_BUFFER_INFO_FLAG_ACTIVE; @@ -1321,10 +1343,25 @@ static int lan743x_tx_get_avail_desc(struct lan743x_tx *tx) return last_head - last_tail - 1; } +void lan743x_tx_set_timestamping_mode(struct lan743x_tx *tx, + bool enable_timestamping, + bool enable_onestep_sync) +{ + if (enable_timestamping) + tx->ts_flags |= TX_TS_FLAG_TIMESTAMPING_ENABLED; + else + tx->ts_flags &= ~TX_TS_FLAG_TIMESTAMPING_ENABLED; + if (enable_onestep_sync) + tx->ts_flags |= TX_TS_FLAG_ONE_STEP_SYNC; + else + tx->ts_flags &= ~TX_TS_FLAG_ONE_STEP_SYNC; +} + static int lan743x_tx_frame_start(struct lan743x_tx *tx, unsigned char *first_buffer, unsigned int first_buffer_length, unsigned int frame_length, + bool time_stamp, bool check_sum) { /* called only from within lan743x_tx_xmit_frame. @@ -1362,6 +1399,8 @@ static int lan743x_tx_frame_start(struct lan743x_tx *tx, TX_DESC_DATA0_DTYPE_DATA_ | TX_DESC_DATA0_FS_ | TX_DESC_DATA0_FCS_; + if (time_stamp) + tx->frame_data0 |= TX_DESC_DATA0_TSE_; if (check_sum) tx->frame_data0 |= TX_DESC_DATA0_ICE_ | @@ -1475,6 +1514,7 @@ static int lan743x_tx_frame_add_fragment(struct lan743x_tx *tx, static void lan743x_tx_frame_end(struct lan743x_tx *tx, struct sk_buff *skb, + bool time_stamp, bool ignore_sync) { /* called only from within lan743x_tx_xmit_frame @@ -1492,6 +1532,8 @@ static void lan743x_tx_frame_end(struct lan743x_tx *tx, tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail]; buffer_info = &tx->buffer_info[tx->frame_tail]; buffer_info->skb = skb; + if (time_stamp) + buffer_info->flags |= TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED; if (ignore_sync) buffer_info->flags |= TX_BUFFER_INFO_FLAG_IGNORE_SYNC; @@ -1520,6 +1562,7 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx, unsigned int frame_length = 0; unsigned int head_length = 0; unsigned long irq_flags = 0; + bool do_timestamp = false; bool ignore_sync = false; int nr_frags = 0; bool gso = false; @@ -1541,6 +1584,14 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx, } /* space available, transmit skb */ + if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + (tx->ts_flags & TX_TS_FLAG_TIMESTAMPING_ENABLED) && + (lan743x_ptp_request_tx_timestamp(tx->adapter))) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + do_timestamp = true; + if (tx->ts_flags & TX_TS_FLAG_ONE_STEP_SYNC) + ignore_sync = true; + } head_length = skb_headlen(skb); frame_length = skb_pagelen(skb); nr_frags = skb_shinfo(skb)->nr_frags; @@ -1554,6 +1605,7 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx, if (lan743x_tx_frame_start(tx, skb->data, head_length, start_frame_length, + do_timestamp, skb->ip_summed == CHECKSUM_PARTIAL)) { dev_kfree_skb(skb); goto unlock; @@ -1581,7 +1633,7 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx, } finish: - lan743x_tx_frame_end(tx, skb, ignore_sync); + lan743x_tx_frame_end(tx, skb, do_timestamp, ignore_sync); unlock: spin_unlock_irqrestore(&tx->ring_lock, irq_flags); @@ -2410,6 +2462,8 @@ static int lan743x_netdev_close(struct net_device *netdev) for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) lan743x_rx_close(&adapter->rx[index]); + lan743x_ptp_close(adapter); + lan743x_phy_close(adapter); lan743x_mac_close(adapter); @@ -2437,6 +2491,10 @@ static int lan743x_netdev_open(struct net_device *netdev) if (ret) goto close_mac; + ret = lan743x_ptp_open(adapter); + if (ret) + goto close_phy; + lan743x_rfe_open(adapter); for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) { @@ -2456,6 +2514,9 @@ close_rx: if (adapter->rx[index].ring_cpu_ptr) lan743x_rx_close(&adapter->rx[index]); } + lan743x_ptp_close(adapter); + +close_phy: lan743x_phy_close(adapter); close_mac: @@ -2483,6 +2544,8 @@ static int lan743x_netdev_ioctl(struct net_device *netdev, { if (!netif_running(netdev)) return -EINVAL; + if (cmd == SIOCSHWTSTAMP) + return lan743x_ptp_ioctl(netdev, ifr, cmd); return phy_mii_ioctl(netdev->phydev, ifr, cmd); } @@ -2607,6 +2670,11 @@ static int lan743x_hardware_init(struct lan743x_adapter *adapter, adapter->intr.irq = adapter->pdev->irq; lan743x_csr_write(adapter, INT_EN_CLR, 0xFFFFFFFF); mutex_init(&adapter->dp_lock); + + ret = lan743x_gpio_init(adapter); + if (ret) + return ret; + ret = lan743x_mac_init(adapter); if (ret) return ret; @@ -2615,6 +2683,10 @@ static int lan743x_hardware_init(struct lan743x_adapter *adapter, if (ret) return ret; + ret = lan743x_ptp_init(adapter); + if (ret) + return ret; + lan743x_rfe_update_mac_address(adapter); ret = lan743x_dmac_init(adapter); diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h index 4fa7a5e027f4..0e82b6368798 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.h +++ b/drivers/net/ethernet/microchip/lan743x_main.h @@ -4,12 +4,17 @@ #ifndef _LAN743X_H #define _LAN743X_H +#include "lan743x_ptp.h" + #define DRIVER_AUTHOR "Bryan Whitehead <Bryan.Whitehead@microchip.com>" #define DRIVER_DESC "LAN743x PCIe Gigabit Ethernet Driver" #define DRIVER_NAME "lan743x" /* Register Definitions */ #define ID_REV (0x00) +#define ID_REV_ID_MASK_ (0xFFFF0000) +#define ID_REV_ID_LAN7430_ (0x74300000) +#define ID_REV_ID_LAN7431_ (0x74310000) #define ID_REV_IS_VALID_CHIP_ID_(id_rev) \ (((id_rev) & 0xFFF00000) == 0x74300000) #define ID_REV_CHIP_REV_MASK_ (0x0000FFFF) @@ -62,6 +67,21 @@ #define E2P_DATA (0x044) +#define GPIO_CFG0 (0x050) +#define GPIO_CFG0_GPIO_DIR_BIT_(bit) BIT(16 + (bit)) +#define GPIO_CFG0_GPIO_DATA_BIT_(bit) BIT(0 + (bit)) + +#define GPIO_CFG1 (0x054) +#define GPIO_CFG1_GPIOEN_BIT_(bit) BIT(16 + (bit)) +#define GPIO_CFG1_GPIOBUF_BIT_(bit) BIT(0 + (bit)) + +#define GPIO_CFG2 (0x058) +#define GPIO_CFG2_1588_POL_BIT_(bit) BIT(0 + (bit)) + +#define GPIO_CFG3 (0x05C) +#define GPIO_CFG3_1588_CH_SEL_BIT_(bit) BIT(16 + (bit)) +#define GPIO_CFG3_1588_OE_BIT_(bit) BIT(0 + (bit)) + #define FCT_RX_CTL (0xAC) #define FCT_RX_CTL_EN_(channel) BIT(28 + (channel)) #define FCT_RX_CTL_DIS_(channel) BIT(24 + (channel)) @@ -193,7 +213,8 @@ #define INT_BIT_DMA_TX_(channel) BIT(16 + (channel)) #define INT_BIT_ALL_TX_ (0x000F0000) #define INT_BIT_SW_GP_ BIT(9) -#define INT_BIT_ALL_OTHER_ (0x00000280) +#define INT_BIT_1588_ BIT(7) +#define INT_BIT_ALL_OTHER_ (INT_BIT_SW_GP_ | INT_BIT_1588_) #define INT_BIT_MAS_ BIT(0) #define INT_SET (0x784) @@ -234,6 +255,71 @@ #define INT_MOD_CFG6 (0x7D8) #define INT_MOD_CFG7 (0x7DC) +#define PTP_CMD_CTL (0x0A00) +#define PTP_CMD_CTL_PTP_CLK_STP_NSEC_ BIT(6) +#define PTP_CMD_CTL_PTP_CLOCK_STEP_SEC_ BIT(5) +#define PTP_CMD_CTL_PTP_CLOCK_LOAD_ BIT(4) +#define PTP_CMD_CTL_PTP_CLOCK_READ_ BIT(3) +#define PTP_CMD_CTL_PTP_ENABLE_ BIT(2) +#define PTP_CMD_CTL_PTP_DISABLE_ BIT(1) +#define PTP_CMD_CTL_PTP_RESET_ BIT(0) +#define PTP_GENERAL_CONFIG (0x0A04) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_X_MASK_(channel) \ + (0x7 << (1 + ((channel) << 2))) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_100NS_ (0) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_10US_ (1) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_100US_ (2) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_1MS_ (3) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_10MS_ (4) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_200MS_ (5) +#define PTP_GENERAL_CONFIG_CLOCK_EVENT_X_SET_(channel, value) \ + (((value) & 0x7) << (1 + ((channel) << 2))) +#define PTP_GENERAL_CONFIG_RELOAD_ADD_X_(channel) (BIT((channel) << 2)) + +#define PTP_INT_STS (0x0A08) +#define PTP_INT_EN_SET (0x0A0C) +#define PTP_INT_EN_CLR (0x0A10) +#define PTP_INT_BIT_TX_SWTS_ERR_ BIT(13) +#define PTP_INT_BIT_TX_TS_ BIT(12) +#define PTP_INT_BIT_TIMER_B_ BIT(1) +#define PTP_INT_BIT_TIMER_A_ BIT(0) + +#define PTP_CLOCK_SEC (0x0A14) +#define PTP_CLOCK_NS (0x0A18) +#define PTP_CLOCK_SUBNS (0x0A1C) +#define PTP_CLOCK_RATE_ADJ (0x0A20) +#define PTP_CLOCK_RATE_ADJ_DIR_ BIT(31) +#define PTP_CLOCK_STEP_ADJ (0x0A2C) +#define PTP_CLOCK_STEP_ADJ_DIR_ BIT(31) +#define PTP_CLOCK_STEP_ADJ_VALUE_MASK_ (0x3FFFFFFF) +#define PTP_CLOCK_TARGET_SEC_X(channel) (0x0A30 + ((channel) << 4)) +#define PTP_CLOCK_TARGET_NS_X(channel) (0x0A34 + ((channel) << 4)) +#define PTP_CLOCK_TARGET_RELOAD_SEC_X(channel) (0x0A38 + ((channel) << 4)) +#define PTP_CLOCK_TARGET_RELOAD_NS_X(channel) (0x0A3C + ((channel) << 4)) +#define PTP_LATENCY (0x0A5C) +#define PTP_LATENCY_TX_SET_(tx_latency) (((u32)(tx_latency)) << 16) +#define PTP_LATENCY_RX_SET_(rx_latency) \ + (((u32)(rx_latency)) & 0x0000FFFF) +#define PTP_CAP_INFO (0x0A60) +#define PTP_CAP_INFO_TX_TS_CNT_GET_(reg_val) (((reg_val) & 0x00000070) >> 4) + +#define PTP_TX_MOD (0x0AA4) +#define PTP_TX_MOD_TX_PTP_SYNC_TS_INSERT_ (0x10000000) + +#define PTP_TX_MOD2 (0x0AA8) +#define PTP_TX_MOD2_TX_PTP_CLR_UDPV4_CHKSUM_ (0x00000001) + +#define PTP_TX_EGRESS_SEC (0x0AAC) +#define PTP_TX_EGRESS_NS (0x0AB0) +#define PTP_TX_EGRESS_NS_CAPTURE_CAUSE_MASK_ (0xC0000000) +#define PTP_TX_EGRESS_NS_CAPTURE_CAUSE_AUTO_ (0x00000000) +#define PTP_TX_EGRESS_NS_CAPTURE_CAUSE_SW_ (0x40000000) +#define PTP_TX_EGRESS_NS_TS_NS_MASK_ (0x3FFFFFFF) + +#define PTP_TX_MSG_HEADER (0x0AB4) +#define PTP_TX_MSG_HEADER_MSG_TYPE_ (0x000F0000) +#define PTP_TX_MSG_HEADER_MSG_TYPE_SYNC_ (0x00000000) + #define DMAC_CFG (0xC00) #define DMAC_CFG_COAL_EN_ BIT(16) #define DMAC_CFG_CH_ARB_SEL_RX_HIGH_ (0x00000000) @@ -542,8 +628,12 @@ struct lan743x_tx_buffer_info; #define TX_FRAME_FLAG_IN_PROGRESS BIT(0) +#define TX_TS_FLAG_TIMESTAMPING_ENABLED BIT(0) +#define TX_TS_FLAG_ONE_STEP_SYNC BIT(1) + struct lan743x_tx { struct lan743x_adapter *adapter; + u32 ts_flags; u32 vector_flags; int channel_number; @@ -570,6 +660,10 @@ struct lan743x_tx { struct sk_buff *overflow_skb; }; +void lan743x_tx_set_timestamping_mode(struct lan743x_tx *tx, + bool enable_timestamping, + bool enable_onestep_sync); + /* RX */ struct lan743x_rx_descriptor; struct lan743x_rx_buffer_info; @@ -610,6 +704,9 @@ struct lan743x_adapter { /* lock, used to prevent concurrent access to data port */ struct mutex dp_lock; + struct lan743x_gpio gpio; + struct lan743x_ptp ptp; + u8 mac_address[ETH_ALEN]; struct lan743x_phy phy; @@ -660,6 +757,7 @@ struct lan743x_adapter { #define TX_DESC_DATA0_IPE_ (0x00200000) #define TX_DESC_DATA0_TPE_ (0x00100000) #define TX_DESC_DATA0_FCS_ (0x00020000) +#define TX_DESC_DATA0_TSE_ (0x00010000) #define TX_DESC_DATA0_BUF_LENGTH_MASK_ (0x0000FFFF) #define TX_DESC_DATA0_EXT_LSO_ (0x00200000) #define TX_DESC_DATA0_EXT_PAY_LENGTH_MASK_ (0x000FFFFF) @@ -673,6 +771,7 @@ struct lan743x_tx_descriptor { } __aligned(DEFAULT_DMA_DESCRIPTOR_SPACING); #define TX_BUFFER_INFO_FLAG_ACTIVE BIT(0) +#define TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED BIT(1) #define TX_BUFFER_INFO_FLAG_IGNORE_SYNC BIT(2) #define TX_BUFFER_INFO_FLAG_SKB_FRAGMENT BIT(3) struct lan743x_tx_buffer_info { diff --git a/drivers/net/ethernet/microchip/lan743x_ptp.c b/drivers/net/ethernet/microchip/lan743x_ptp.c new file mode 100644 index 000000000000..029a2af90d5e --- /dev/null +++ b/drivers/net/ethernet/microchip/lan743x_ptp.c @@ -0,0 +1,1163 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (C) 2018 Microchip Technology Inc. */ + +#include <linux/netdevice.h> +#include "lan743x_main.h" + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/net_tstamp.h> + +#include "lan743x_ptp.h" + +#define LAN743X_NUMBER_OF_GPIO (12) +#define LAN743X_PTP_MAX_FREQ_ADJ_IN_PPB (31249999) +#define LAN743X_PTP_MAX_FINE_ADJ_IN_SCALED_PPM (2047999934) + +static bool lan743x_ptp_is_enabled(struct lan743x_adapter *adapter); +static void lan743x_ptp_enable(struct lan743x_adapter *adapter); +static void lan743x_ptp_disable(struct lan743x_adapter *adapter); +static void lan743x_ptp_reset(struct lan743x_adapter *adapter); +static void lan743x_ptp_clock_set(struct lan743x_adapter *adapter, + u32 seconds, u32 nano_seconds, + u32 sub_nano_seconds); + +int lan743x_gpio_init(struct lan743x_adapter *adapter) +{ + struct lan743x_gpio *gpio = &adapter->gpio; + + spin_lock_init(&gpio->gpio_lock); + + gpio->gpio_cfg0 = 0; /* set all direction to input, data = 0 */ + gpio->gpio_cfg1 = 0x0FFF0000;/* disable all gpio, set to open drain */ + gpio->gpio_cfg2 = 0;/* set all to 1588 low polarity level */ + gpio->gpio_cfg3 = 0;/* disable all 1588 output */ + lan743x_csr_write(adapter, GPIO_CFG0, gpio->gpio_cfg0); + lan743x_csr_write(adapter, GPIO_CFG1, gpio->gpio_cfg1); + lan743x_csr_write(adapter, GPIO_CFG2, gpio->gpio_cfg2); + lan743x_csr_write(adapter, GPIO_CFG3, gpio->gpio_cfg3); + + return 0; +} + +static void lan743x_ptp_wait_till_cmd_done(struct lan743x_adapter *adapter, + u32 bit_mask) +{ + int timeout = 1000; + u32 data = 0; + + while (timeout && + (data = (lan743x_csr_read(adapter, PTP_CMD_CTL) & + bit_mask))) { + usleep_range(1000, 20000); + timeout--; + } + if (data) { + netif_err(adapter, drv, adapter->netdev, + "timeout waiting for cmd to be done, cmd = 0x%08X\n", + bit_mask); + } +} + +static void lan743x_ptp_tx_ts_enqueue_ts(struct lan743x_adapter *adapter, + u32 seconds, u32 nano_seconds, + u32 header) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + spin_lock_bh(&ptp->tx_ts_lock); + if (ptp->tx_ts_queue_size < LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS) { + ptp->tx_ts_seconds_queue[ptp->tx_ts_queue_size] = seconds; + ptp->tx_ts_nseconds_queue[ptp->tx_ts_queue_size] = nano_seconds; + ptp->tx_ts_header_queue[ptp->tx_ts_queue_size] = header; + ptp->tx_ts_queue_size++; + } else { + netif_err(adapter, drv, adapter->netdev, + "tx ts queue overflow\n"); + } + spin_unlock_bh(&ptp->tx_ts_lock); +} + +static void lan743x_ptp_tx_ts_complete(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + struct skb_shared_hwtstamps tstamps; + u32 header, nseconds, seconds; + bool ignore_sync = false; + struct sk_buff *skb; + int c, i; + + spin_lock_bh(&ptp->tx_ts_lock); + c = ptp->tx_ts_skb_queue_size; + + if (c > ptp->tx_ts_queue_size) + c = ptp->tx_ts_queue_size; + if (c <= 0) + goto done; + + for (i = 0; i < c; i++) { + ignore_sync = ((ptp->tx_ts_ignore_sync_queue & + BIT(i)) != 0); + skb = ptp->tx_ts_skb_queue[i]; + nseconds = ptp->tx_ts_nseconds_queue[i]; + seconds = ptp->tx_ts_seconds_queue[i]; + header = ptp->tx_ts_header_queue[i]; + + memset(&tstamps, 0, sizeof(tstamps)); + tstamps.hwtstamp = ktime_set(seconds, nseconds); + if (!ignore_sync || + ((header & PTP_TX_MSG_HEADER_MSG_TYPE_) != + PTP_TX_MSG_HEADER_MSG_TYPE_SYNC_)) + skb_tstamp_tx(skb, &tstamps); + + dev_kfree_skb(skb); + + ptp->tx_ts_skb_queue[i] = NULL; + ptp->tx_ts_seconds_queue[i] = 0; + ptp->tx_ts_nseconds_queue[i] = 0; + ptp->tx_ts_header_queue[i] = 0; + } + + /* shift queue */ + ptp->tx_ts_ignore_sync_queue >>= c; + for (i = c; i < LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS; i++) { + ptp->tx_ts_skb_queue[i - c] = ptp->tx_ts_skb_queue[i]; + ptp->tx_ts_seconds_queue[i - c] = ptp->tx_ts_seconds_queue[i]; + ptp->tx_ts_nseconds_queue[i - c] = ptp->tx_ts_nseconds_queue[i]; + ptp->tx_ts_header_queue[i - c] = ptp->tx_ts_header_queue[i]; + + ptp->tx_ts_skb_queue[i] = NULL; + ptp->tx_ts_seconds_queue[i] = 0; + ptp->tx_ts_nseconds_queue[i] = 0; + ptp->tx_ts_header_queue[i] = 0; + } + ptp->tx_ts_skb_queue_size -= c; + ptp->tx_ts_queue_size -= c; +done: + ptp->pending_tx_timestamps -= c; + spin_unlock_bh(&ptp->tx_ts_lock); +} + +#ifdef CONFIG_PTP_1588_CLOCK +static int lan743x_ptp_reserve_event_ch(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + int result = -ENODEV; + int index = 0; + + mutex_lock(&ptp->command_lock); + for (index = 0; index < LAN743X_PTP_NUMBER_OF_EVENT_CHANNELS; index++) { + if (!(test_bit(index, &ptp->used_event_ch))) { + ptp->used_event_ch |= BIT(index); + result = index; + break; + } + } + mutex_unlock(&ptp->command_lock); + return result; +} + +static void lan743x_ptp_release_event_ch(struct lan743x_adapter *adapter, + int event_channel) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + mutex_lock(&ptp->command_lock); + if (test_bit(event_channel, &ptp->used_event_ch)) { + ptp->used_event_ch &= ~BIT(event_channel); + } else { + netif_warn(adapter, drv, adapter->netdev, + "attempted release on a not used event_channel = %d\n", + event_channel); + } + mutex_unlock(&ptp->command_lock); +} + +static void lan743x_ptp_clock_get(struct lan743x_adapter *adapter, + u32 *seconds, u32 *nano_seconds, + u32 *sub_nano_seconds); +static void lan743x_ptp_clock_step(struct lan743x_adapter *adapter, + s64 time_step_ns); + +static int lan743x_gpio_rsrv_ptp_out(struct lan743x_adapter *adapter, + int bit, int ptp_channel) +{ + struct lan743x_gpio *gpio = &adapter->gpio; + unsigned long irq_flags = 0; + int bit_mask = BIT(bit); + int ret = -EBUSY; + + spin_lock_irqsave(&gpio->gpio_lock, irq_flags); + + if (!(gpio->used_bits & bit_mask)) { + gpio->used_bits |= bit_mask; + gpio->output_bits |= bit_mask; + gpio->ptp_bits |= bit_mask; + + /* set as output, and zero initial value */ + gpio->gpio_cfg0 |= GPIO_CFG0_GPIO_DIR_BIT_(bit); + gpio->gpio_cfg0 &= ~GPIO_CFG0_GPIO_DATA_BIT_(bit); + lan743x_csr_write(adapter, GPIO_CFG0, gpio->gpio_cfg0); + + /* enable gpio, and set buffer type to push pull */ + gpio->gpio_cfg1 &= ~GPIO_CFG1_GPIOEN_BIT_(bit); + gpio->gpio_cfg1 |= GPIO_CFG1_GPIOBUF_BIT_(bit); + lan743x_csr_write(adapter, GPIO_CFG1, gpio->gpio_cfg1); + + /* set 1588 polarity to high */ + gpio->gpio_cfg2 |= GPIO_CFG2_1588_POL_BIT_(bit); + lan743x_csr_write(adapter, GPIO_CFG2, gpio->gpio_cfg2); + + if (!ptp_channel) { + /* use channel A */ + gpio->gpio_cfg3 &= ~GPIO_CFG3_1588_CH_SEL_BIT_(bit); + } else { + /* use channel B */ + gpio->gpio_cfg3 |= GPIO_CFG3_1588_CH_SEL_BIT_(bit); + } + gpio->gpio_cfg3 |= GPIO_CFG3_1588_OE_BIT_(bit); + lan743x_csr_write(adapter, GPIO_CFG3, gpio->gpio_cfg3); + + ret = bit; + } + spin_unlock_irqrestore(&gpio->gpio_lock, irq_flags); + return ret; +} + +static void lan743x_gpio_release(struct lan743x_adapter *adapter, int bit) +{ + struct lan743x_gpio *gpio = &adapter->gpio; + unsigned long irq_flags = 0; + int bit_mask = BIT(bit); + + spin_lock_irqsave(&gpio->gpio_lock, irq_flags); + if (gpio->used_bits & bit_mask) { + gpio->used_bits &= ~bit_mask; + if (gpio->output_bits & bit_mask) { + gpio->output_bits &= ~bit_mask; + + if (gpio->ptp_bits & bit_mask) { + gpio->ptp_bits &= ~bit_mask; + /* disable ptp output */ + gpio->gpio_cfg3 &= ~GPIO_CFG3_1588_OE_BIT_(bit); + lan743x_csr_write(adapter, GPIO_CFG3, + gpio->gpio_cfg3); + } + /* release gpio output */ + + /* disable gpio */ + gpio->gpio_cfg1 |= GPIO_CFG1_GPIOEN_BIT_(bit); + gpio->gpio_cfg1 &= ~GPIO_CFG1_GPIOBUF_BIT_(bit); + lan743x_csr_write(adapter, GPIO_CFG1, gpio->gpio_cfg1); + + /* reset back to input */ + gpio->gpio_cfg0 &= ~GPIO_CFG0_GPIO_DIR_BIT_(bit); + gpio->gpio_cfg0 &= ~GPIO_CFG0_GPIO_DATA_BIT_(bit); + lan743x_csr_write(adapter, GPIO_CFG0, gpio->gpio_cfg0); + } + } + spin_unlock_irqrestore(&gpio->gpio_lock, irq_flags); +} + +static int lan743x_ptpci_adjfine(struct ptp_clock_info *ptpci, long scaled_ppm) +{ + struct lan743x_ptp *ptp = + container_of(ptpci, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(ptp, struct lan743x_adapter, ptp); + u32 lan743x_rate_adj = 0; + bool positive = true; + u64 u64_delta = 0; + + if ((scaled_ppm < (-LAN743X_PTP_MAX_FINE_ADJ_IN_SCALED_PPM)) || + scaled_ppm > LAN743X_PTP_MAX_FINE_ADJ_IN_SCALED_PPM) { + return -EINVAL; + } + if (scaled_ppm > 0) { + u64_delta = (u64)scaled_ppm; + positive = true; + } else { + u64_delta = (u64)(-scaled_ppm); + positive = false; + } + u64_delta = (u64_delta << 19); + lan743x_rate_adj = div_u64(u64_delta, 1000000); + + if (positive) + lan743x_rate_adj |= PTP_CLOCK_RATE_ADJ_DIR_; + + lan743x_csr_write(adapter, PTP_CLOCK_RATE_ADJ, + lan743x_rate_adj); + + return 0; +} + +static int lan743x_ptpci_adjfreq(struct ptp_clock_info *ptpci, s32 delta_ppb) +{ + struct lan743x_ptp *ptp = + container_of(ptpci, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(ptp, struct lan743x_adapter, ptp); + u32 lan743x_rate_adj = 0; + bool positive = true; + u32 u32_delta = 0; + u64 u64_delta = 0; + + if ((delta_ppb < (-LAN743X_PTP_MAX_FREQ_ADJ_IN_PPB)) || + delta_ppb > LAN743X_PTP_MAX_FREQ_ADJ_IN_PPB) { + return -EINVAL; + } + if (delta_ppb > 0) { + u32_delta = (u32)delta_ppb; + positive = true; + } else { + u32_delta = (u32)(-delta_ppb); + positive = false; + } + u64_delta = (((u64)u32_delta) << 35); + lan743x_rate_adj = div_u64(u64_delta, 1000000000); + + if (positive) + lan743x_rate_adj |= PTP_CLOCK_RATE_ADJ_DIR_; + + lan743x_csr_write(adapter, PTP_CLOCK_RATE_ADJ, + lan743x_rate_adj); + + return 0; +} + +static int lan743x_ptpci_adjtime(struct ptp_clock_info *ptpci, s64 delta) +{ + struct lan743x_ptp *ptp = + container_of(ptpci, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(ptp, struct lan743x_adapter, ptp); + + lan743x_ptp_clock_step(adapter, delta); + + return 0; +} + +static int lan743x_ptpci_gettime64(struct ptp_clock_info *ptpci, + struct timespec64 *ts) +{ + struct lan743x_ptp *ptp = + container_of(ptpci, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(ptp, struct lan743x_adapter, ptp); + u32 nano_seconds = 0; + u32 seconds = 0; + + lan743x_ptp_clock_get(adapter, &seconds, &nano_seconds, NULL); + ts->tv_sec = seconds; + ts->tv_nsec = nano_seconds; + + return 0; +} + +static int lan743x_ptpci_settime64(struct ptp_clock_info *ptpci, + const struct timespec64 *ts) +{ + struct lan743x_ptp *ptp = + container_of(ptpci, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(ptp, struct lan743x_adapter, ptp); + u32 nano_seconds = 0; + u32 seconds = 0; + + if (ts) { + if (ts->tv_sec > 0xFFFFFFFFLL || + ts->tv_sec < 0) { + netif_warn(adapter, drv, adapter->netdev, + "ts->tv_sec out of range, %lld\n", + ts->tv_sec); + return -ERANGE; + } + if (ts->tv_nsec >= 1000000000L || + ts->tv_nsec < 0) { + netif_warn(adapter, drv, adapter->netdev, + "ts->tv_nsec out of range, %ld\n", + ts->tv_nsec); + return -ERANGE; + } + seconds = ts->tv_sec; + nano_seconds = ts->tv_nsec; + lan743x_ptp_clock_set(adapter, seconds, nano_seconds, 0); + } else { + netif_warn(adapter, drv, adapter->netdev, "ts == NULL\n"); + return -EINVAL; + } + + return 0; +} + +static void lan743x_ptp_perout_off(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + u32 general_config = 0; + + if (ptp->perout_gpio_bit >= 0) { + lan743x_gpio_release(adapter, ptp->perout_gpio_bit); + ptp->perout_gpio_bit = -1; + } + + if (ptp->perout_event_ch >= 0) { + /* set target to far in the future, effectively disabling it */ + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_SEC_X(ptp->perout_event_ch), + 0xFFFF0000); + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_NS_X(ptp->perout_event_ch), + 0); + + general_config = lan743x_csr_read(adapter, PTP_GENERAL_CONFIG); + general_config |= PTP_GENERAL_CONFIG_RELOAD_ADD_X_ + (ptp->perout_event_ch); + lan743x_csr_write(adapter, PTP_GENERAL_CONFIG, general_config); + lan743x_ptp_release_event_ch(adapter, ptp->perout_event_ch); + ptp->perout_event_ch = -1; + } +} + +static int lan743x_ptp_perout(struct lan743x_adapter *adapter, int on, + struct ptp_perout_request *perout) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + u32 period_sec = 0, period_nsec = 0; + u32 start_sec = 0, start_nsec = 0; + u32 general_config = 0; + int pulse_width = 0; + int perout_bit = 0; + + if (!on) { + lan743x_ptp_perout_off(adapter); + return 0; + } + + if (ptp->perout_event_ch >= 0 || + ptp->perout_gpio_bit >= 0) { + /* already on, turn off first */ + lan743x_ptp_perout_off(adapter); + } + + ptp->perout_event_ch = lan743x_ptp_reserve_event_ch(adapter); + if (ptp->perout_event_ch < 0) { + netif_warn(adapter, drv, adapter->netdev, + "Failed to reserve event channel for PEROUT\n"); + goto failed; + } + + switch (adapter->csr.id_rev & ID_REV_ID_MASK_) { + case ID_REV_ID_LAN7430_: + perout_bit = 2;/* GPIO 2 is preferred on EVB LAN7430 */ + break; + case ID_REV_ID_LAN7431_: + perout_bit = 4;/* GPIO 4 is preferred on EVB LAN7431 */ + break; + } + + ptp->perout_gpio_bit = lan743x_gpio_rsrv_ptp_out(adapter, + perout_bit, + ptp->perout_event_ch); + + if (ptp->perout_gpio_bit < 0) { + netif_warn(adapter, drv, adapter->netdev, + "Failed to reserve gpio %d for PEROUT\n", + perout_bit); + goto failed; + } + + start_sec = perout->start.sec; + start_sec += perout->start.nsec / 1000000000; + start_nsec = perout->start.nsec % 1000000000; + + period_sec = perout->period.sec; + period_sec += perout->period.nsec / 1000000000; + period_nsec = perout->period.nsec % 1000000000; + + if (period_sec == 0) { + if (period_nsec >= 400000000) { + pulse_width = PTP_GENERAL_CONFIG_CLOCK_EVENT_200MS_; + } else if (period_nsec >= 20000000) { + pulse_width = PTP_GENERAL_CONFIG_CLOCK_EVENT_10MS_; + } else if (period_nsec >= 2000000) { + pulse_width = PTP_GENERAL_CONFIG_CLOCK_EVENT_1MS_; + } else if (period_nsec >= 200000) { + pulse_width = PTP_GENERAL_CONFIG_CLOCK_EVENT_100US_; + } else if (period_nsec >= 20000) { + pulse_width = PTP_GENERAL_CONFIG_CLOCK_EVENT_10US_; + } else if (period_nsec >= 200) { + pulse_width = PTP_GENERAL_CONFIG_CLOCK_EVENT_100NS_; + } else { + netif_warn(adapter, drv, adapter->netdev, + "perout period too small, minimum is 200nS\n"); + goto failed; + } + } else { + pulse_width = PTP_GENERAL_CONFIG_CLOCK_EVENT_200MS_; + } + + /* turn off by setting target far in future */ + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_SEC_X(ptp->perout_event_ch), + 0xFFFF0000); + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_NS_X(ptp->perout_event_ch), 0); + + /* Configure to pulse every period */ + general_config = lan743x_csr_read(adapter, PTP_GENERAL_CONFIG); + general_config &= ~(PTP_GENERAL_CONFIG_CLOCK_EVENT_X_MASK_ + (ptp->perout_event_ch)); + general_config |= PTP_GENERAL_CONFIG_CLOCK_EVENT_X_SET_ + (ptp->perout_event_ch, pulse_width); + general_config &= ~PTP_GENERAL_CONFIG_RELOAD_ADD_X_ + (ptp->perout_event_ch); + lan743x_csr_write(adapter, PTP_GENERAL_CONFIG, general_config); + + /* set the reload to one toggle cycle */ + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_RELOAD_SEC_X(ptp->perout_event_ch), + period_sec); + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_RELOAD_NS_X(ptp->perout_event_ch), + period_nsec); + + /* set the start time */ + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_SEC_X(ptp->perout_event_ch), + start_sec); + lan743x_csr_write(adapter, + PTP_CLOCK_TARGET_NS_X(ptp->perout_event_ch), + start_nsec); + + return 0; + +failed: + lan743x_ptp_perout_off(adapter); + return -ENODEV; +} + +static int lan743x_ptpci_enable(struct ptp_clock_info *ptpci, + struct ptp_clock_request *request, int on) +{ + struct lan743x_ptp *ptp = + container_of(ptpci, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(ptp, struct lan743x_adapter, ptp); + + if (request) { + switch (request->type) { + case PTP_CLK_REQ_EXTTS: + return -EINVAL; + case PTP_CLK_REQ_PEROUT: + if (request->perout.index == 0) + return lan743x_ptp_perout(adapter, on, + &request->perout); + return -EINVAL; + case PTP_CLK_REQ_PPS: + return -EINVAL; + default: + netif_err(adapter, drv, adapter->netdev, + "request->type == %d, Unknown\n", + request->type); + break; + } + } else { + netif_err(adapter, drv, adapter->netdev, "request == NULL\n"); + } + return 0; +} + +static long lan743x_ptpci_do_aux_work(struct ptp_clock_info *ptpci) +{ + struct lan743x_ptp *ptp = + container_of(ptpci, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(ptp, struct lan743x_adapter, ptp); + u32 cap_info, cause, header, nsec, seconds; + bool new_timestamp_available = false; + int count = 0; + + while ((count < 100) && + (lan743x_csr_read(adapter, PTP_INT_STS) & PTP_INT_BIT_TX_TS_)) { + count++; + cap_info = lan743x_csr_read(adapter, PTP_CAP_INFO); + + if (PTP_CAP_INFO_TX_TS_CNT_GET_(cap_info) > 0) { + seconds = lan743x_csr_read(adapter, + PTP_TX_EGRESS_SEC); + nsec = lan743x_csr_read(adapter, PTP_TX_EGRESS_NS); + cause = (nsec & + PTP_TX_EGRESS_NS_CAPTURE_CAUSE_MASK_); + header = lan743x_csr_read(adapter, + PTP_TX_MSG_HEADER); + + if (cause == PTP_TX_EGRESS_NS_CAPTURE_CAUSE_SW_) { + nsec &= PTP_TX_EGRESS_NS_TS_NS_MASK_; + lan743x_ptp_tx_ts_enqueue_ts(adapter, + seconds, nsec, + header); + new_timestamp_available = true; + } else if (cause == + PTP_TX_EGRESS_NS_CAPTURE_CAUSE_AUTO_) { + netif_err(adapter, drv, adapter->netdev, + "Auto capture cause not supported\n"); + } else { + netif_warn(adapter, drv, adapter->netdev, + "unknown tx timestamp capture cause\n"); + } + } else { + netif_warn(adapter, drv, adapter->netdev, + "TX TS INT but no TX TS CNT\n"); + } + lan743x_csr_write(adapter, PTP_INT_STS, PTP_INT_BIT_TX_TS_); + } + + if (new_timestamp_available) + lan743x_ptp_tx_ts_complete(adapter); + + lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_1588_); + + return -1; +} + +static void lan743x_ptp_clock_get(struct lan743x_adapter *adapter, + u32 *seconds, u32 *nano_seconds, + u32 *sub_nano_seconds) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + mutex_lock(&ptp->command_lock); + + lan743x_csr_write(adapter, PTP_CMD_CTL, PTP_CMD_CTL_PTP_CLOCK_READ_); + lan743x_ptp_wait_till_cmd_done(adapter, PTP_CMD_CTL_PTP_CLOCK_READ_); + + if (seconds) + (*seconds) = lan743x_csr_read(adapter, PTP_CLOCK_SEC); + + if (nano_seconds) + (*nano_seconds) = lan743x_csr_read(adapter, PTP_CLOCK_NS); + + if (sub_nano_seconds) + (*sub_nano_seconds) = + lan743x_csr_read(adapter, PTP_CLOCK_SUBNS); + + mutex_unlock(&ptp->command_lock); +} + +static void lan743x_ptp_clock_step(struct lan743x_adapter *adapter, + s64 time_step_ns) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + u32 nano_seconds_step = 0; + u64 abs_time_step_ns = 0; + u32 unsigned_seconds = 0; + u32 nano_seconds = 0; + u32 remainder = 0; + s32 seconds = 0; + + if (time_step_ns > 15000000000LL) { + /* convert to clock set */ + lan743x_ptp_clock_get(adapter, &unsigned_seconds, + &nano_seconds, NULL); + unsigned_seconds += div_u64_rem(time_step_ns, 1000000000LL, + &remainder); + nano_seconds += remainder; + if (nano_seconds >= 1000000000) { + unsigned_seconds++; + nano_seconds -= 1000000000; + } + lan743x_ptp_clock_set(adapter, unsigned_seconds, + nano_seconds, 0); + return; + } else if (time_step_ns < -15000000000LL) { + /* convert to clock set */ + time_step_ns = -time_step_ns; + + lan743x_ptp_clock_get(adapter, &unsigned_seconds, + &nano_seconds, NULL); + unsigned_seconds -= div_u64_rem(time_step_ns, 1000000000LL, + &remainder); + nano_seconds_step = remainder; + if (nano_seconds < nano_seconds_step) { + unsigned_seconds--; + nano_seconds += 1000000000; + } + nano_seconds -= nano_seconds_step; + lan743x_ptp_clock_set(adapter, unsigned_seconds, + nano_seconds, 0); + return; + } + + /* do clock step */ + if (time_step_ns >= 0) { + abs_time_step_ns = (u64)(time_step_ns); + seconds = (s32)div_u64_rem(abs_time_step_ns, 1000000000, + &remainder); + nano_seconds = (u32)remainder; + } else { + abs_time_step_ns = (u64)(-time_step_ns); + seconds = -((s32)div_u64_rem(abs_time_step_ns, 1000000000, + &remainder)); + nano_seconds = (u32)remainder; + if (nano_seconds > 0) { + /* subtracting nano seconds is not allowed + * convert to subtracting from seconds, + * and adding to nanoseconds + */ + seconds--; + nano_seconds = (1000000000 - nano_seconds); + } + } + + if (nano_seconds > 0) { + /* add 8 ns to cover the likely normal increment */ + nano_seconds += 8; + } + + if (nano_seconds >= 1000000000) { + /* carry into seconds */ + seconds++; + nano_seconds -= 1000000000; + } + + while (seconds) { + mutex_lock(&ptp->command_lock); + if (seconds > 0) { + u32 adjustment_value = (u32)seconds; + + if (adjustment_value > 0xF) + adjustment_value = 0xF; + lan743x_csr_write(adapter, PTP_CLOCK_STEP_ADJ, + PTP_CLOCK_STEP_ADJ_DIR_ | + adjustment_value); + seconds -= ((s32)adjustment_value); + } else { + u32 adjustment_value = (u32)(-seconds); + + if (adjustment_value > 0xF) + adjustment_value = 0xF; + lan743x_csr_write(adapter, PTP_CLOCK_STEP_ADJ, + adjustment_value); + seconds += ((s32)adjustment_value); + } + lan743x_csr_write(adapter, PTP_CMD_CTL, + PTP_CMD_CTL_PTP_CLOCK_STEP_SEC_); + lan743x_ptp_wait_till_cmd_done(adapter, + PTP_CMD_CTL_PTP_CLOCK_STEP_SEC_); + mutex_unlock(&ptp->command_lock); + } + if (nano_seconds) { + mutex_lock(&ptp->command_lock); + lan743x_csr_write(adapter, PTP_CLOCK_STEP_ADJ, + PTP_CLOCK_STEP_ADJ_DIR_ | + (nano_seconds & + PTP_CLOCK_STEP_ADJ_VALUE_MASK_)); + lan743x_csr_write(adapter, PTP_CMD_CTL, + PTP_CMD_CTL_PTP_CLK_STP_NSEC_); + lan743x_ptp_wait_till_cmd_done(adapter, + PTP_CMD_CTL_PTP_CLK_STP_NSEC_); + mutex_unlock(&ptp->command_lock); + } +} +#endif /* CONFIG_PTP_1588_CLOCK */ + +void lan743x_ptp_isr(void *context) +{ + struct lan743x_adapter *adapter = (struct lan743x_adapter *)context; + struct lan743x_ptp *ptp = NULL; + int enable_flag = 1; + u32 ptp_int_sts = 0; + + ptp = &adapter->ptp; + + lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_1588_); + + ptp_int_sts = lan743x_csr_read(adapter, PTP_INT_STS); + ptp_int_sts &= lan743x_csr_read(adapter, PTP_INT_EN_SET); + + if (ptp_int_sts & PTP_INT_BIT_TX_TS_) { + ptp_schedule_worker(ptp->ptp_clock, 0); + enable_flag = 0;/* tasklet will re-enable later */ + } + if (ptp_int_sts & PTP_INT_BIT_TX_SWTS_ERR_) { + netif_err(adapter, drv, adapter->netdev, + "PTP TX Software Timestamp Error\n"); + /* clear int status bit */ + lan743x_csr_write(adapter, PTP_INT_STS, + PTP_INT_BIT_TX_SWTS_ERR_); + } + if (ptp_int_sts & PTP_INT_BIT_TIMER_B_) { + /* clear int status bit */ + lan743x_csr_write(adapter, PTP_INT_STS, + PTP_INT_BIT_TIMER_B_); + } + if (ptp_int_sts & PTP_INT_BIT_TIMER_A_) { + /* clear int status bit */ + lan743x_csr_write(adapter, PTP_INT_STS, + PTP_INT_BIT_TIMER_A_); + } + + if (enable_flag) { + /* re-enable isr */ + lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_1588_); + } +} + +static void lan743x_ptp_tx_ts_enqueue_skb(struct lan743x_adapter *adapter, + struct sk_buff *skb, bool ignore_sync) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + spin_lock_bh(&ptp->tx_ts_lock); + if (ptp->tx_ts_skb_queue_size < LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS) { + ptp->tx_ts_skb_queue[ptp->tx_ts_skb_queue_size] = skb; + if (ignore_sync) + ptp->tx_ts_ignore_sync_queue |= + BIT(ptp->tx_ts_skb_queue_size); + ptp->tx_ts_skb_queue_size++; + } else { + /* this should never happen, so long as the tx channel + * calls and honors the result from + * lan743x_ptp_request_tx_timestamp + */ + netif_err(adapter, drv, adapter->netdev, + "tx ts skb queue overflow\n"); + dev_kfree_skb(skb); + } + spin_unlock_bh(&ptp->tx_ts_lock); +} + +static void lan743x_ptp_sync_to_system_clock(struct lan743x_adapter *adapter) +{ + struct timespec64 ts; + + memset(&ts, 0, sizeof(ts)); + timekeeping_clocktai64(&ts); + + lan743x_ptp_clock_set(adapter, ts.tv_sec, ts.tv_nsec, 0); +} + +void lan743x_ptp_update_latency(struct lan743x_adapter *adapter, + u32 link_speed) +{ + switch (link_speed) { + case 10: + lan743x_csr_write(adapter, PTP_LATENCY, + PTP_LATENCY_TX_SET_(0) | + PTP_LATENCY_RX_SET_(0)); + break; + case 100: + lan743x_csr_write(adapter, PTP_LATENCY, + PTP_LATENCY_TX_SET_(181) | + PTP_LATENCY_RX_SET_(594)); + break; + case 1000: + lan743x_csr_write(adapter, PTP_LATENCY, + PTP_LATENCY_TX_SET_(30) | + PTP_LATENCY_RX_SET_(525)); + break; + } +} + +int lan743x_ptp_init(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + mutex_init(&ptp->command_lock); + spin_lock_init(&ptp->tx_ts_lock); + ptp->used_event_ch = 0; + ptp->perout_event_ch = -1; + ptp->perout_gpio_bit = -1; + return 0; +} + +int lan743x_ptp_open(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + int ret = -ENODEV; + u32 temp; + + lan743x_ptp_reset(adapter); + lan743x_ptp_sync_to_system_clock(adapter); + temp = lan743x_csr_read(adapter, PTP_TX_MOD2); + temp |= PTP_TX_MOD2_TX_PTP_CLR_UDPV4_CHKSUM_; + lan743x_csr_write(adapter, PTP_TX_MOD2, temp); + lan743x_ptp_enable(adapter); + lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_1588_); + lan743x_csr_write(adapter, PTP_INT_EN_SET, + PTP_INT_BIT_TX_SWTS_ERR_ | PTP_INT_BIT_TX_TS_); + ptp->flags |= PTP_FLAG_ISR_ENABLED; + +#ifdef CONFIG_PTP_1588_CLOCK + snprintf(ptp->pin_config[0].name, 32, "lan743x_ptp_pin_0"); + ptp->pin_config[0].index = 0; + ptp->pin_config[0].func = PTP_PF_PEROUT; + ptp->pin_config[0].chan = 0; + + ptp->ptp_clock_info.owner = THIS_MODULE; + snprintf(ptp->ptp_clock_info.name, 16, "%pm", + adapter->netdev->dev_addr); + ptp->ptp_clock_info.max_adj = LAN743X_PTP_MAX_FREQ_ADJ_IN_PPB; + ptp->ptp_clock_info.n_alarm = 0; + ptp->ptp_clock_info.n_ext_ts = 0; + ptp->ptp_clock_info.n_per_out = 1; + ptp->ptp_clock_info.n_pins = 0; + ptp->ptp_clock_info.pps = 0; + ptp->ptp_clock_info.pin_config = NULL; + ptp->ptp_clock_info.adjfine = lan743x_ptpci_adjfine; + ptp->ptp_clock_info.adjfreq = lan743x_ptpci_adjfreq; + ptp->ptp_clock_info.adjtime = lan743x_ptpci_adjtime; + ptp->ptp_clock_info.gettime64 = lan743x_ptpci_gettime64; + ptp->ptp_clock_info.getcrosststamp = NULL; + ptp->ptp_clock_info.settime64 = lan743x_ptpci_settime64; + ptp->ptp_clock_info.enable = lan743x_ptpci_enable; + ptp->ptp_clock_info.do_aux_work = lan743x_ptpci_do_aux_work; + ptp->ptp_clock_info.verify = NULL; + + ptp->ptp_clock = ptp_clock_register(&ptp->ptp_clock_info, + &adapter->pdev->dev); + + if (IS_ERR(ptp->ptp_clock)) { + netif_err(adapter, ifup, adapter->netdev, + "ptp_clock_register failed\n"); + goto done; + } + ptp->flags |= PTP_FLAG_PTP_CLOCK_REGISTERED; + netif_info(adapter, ifup, adapter->netdev, + "successfully registered ptp clock\n"); + + return 0; +done: + lan743x_ptp_close(adapter); + return ret; +#else + return 0; +#endif +} + +void lan743x_ptp_close(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + int index; + +#ifdef CONFIG_PTP_1588_CLOCK + if (ptp->flags & PTP_FLAG_PTP_CLOCK_REGISTERED) { + ptp_clock_unregister(ptp->ptp_clock); + ptp->ptp_clock = NULL; + ptp->flags &= ~PTP_FLAG_PTP_CLOCK_REGISTERED; + netif_info(adapter, drv, adapter->netdev, + "ptp clock unregister\n"); + } +#endif + + if (ptp->flags & PTP_FLAG_ISR_ENABLED) { + lan743x_csr_write(adapter, PTP_INT_EN_CLR, + PTP_INT_BIT_TX_SWTS_ERR_ | + PTP_INT_BIT_TX_TS_); + lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_1588_); + ptp->flags &= ~PTP_FLAG_ISR_ENABLED; + } + + /* clean up pending timestamp requests */ + lan743x_ptp_tx_ts_complete(adapter); + spin_lock_bh(&ptp->tx_ts_lock); + for (index = 0; + index < LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS; + index++) { + struct sk_buff *skb = ptp->tx_ts_skb_queue[index]; + + if (skb) + dev_kfree_skb(skb); + ptp->tx_ts_skb_queue[index] = NULL; + ptp->tx_ts_seconds_queue[index] = 0; + ptp->tx_ts_nseconds_queue[index] = 0; + } + ptp->tx_ts_skb_queue_size = 0; + ptp->tx_ts_queue_size = 0; + ptp->pending_tx_timestamps = 0; + spin_unlock_bh(&ptp->tx_ts_lock); + + lan743x_ptp_disable(adapter); +} + +void lan743x_ptp_set_sync_ts_insert(struct lan743x_adapter *adapter, + bool ts_insert_enable) +{ + u32 ptp_tx_mod = lan743x_csr_read(adapter, PTP_TX_MOD); + + if (ts_insert_enable) + ptp_tx_mod |= PTP_TX_MOD_TX_PTP_SYNC_TS_INSERT_; + else + ptp_tx_mod &= ~PTP_TX_MOD_TX_PTP_SYNC_TS_INSERT_; + + lan743x_csr_write(adapter, PTP_TX_MOD, ptp_tx_mod); +} + +static bool lan743x_ptp_is_enabled(struct lan743x_adapter *adapter) +{ + if (lan743x_csr_read(adapter, PTP_CMD_CTL) & PTP_CMD_CTL_PTP_ENABLE_) + return true; + return false; +} + +static void lan743x_ptp_enable(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + mutex_lock(&ptp->command_lock); + + if (lan743x_ptp_is_enabled(adapter)) { + netif_warn(adapter, drv, adapter->netdev, + "PTP already enabled\n"); + goto done; + } + lan743x_csr_write(adapter, PTP_CMD_CTL, PTP_CMD_CTL_PTP_ENABLE_); +done: + mutex_unlock(&ptp->command_lock); +} + +static void lan743x_ptp_disable(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + mutex_lock(&ptp->command_lock); + if (!lan743x_ptp_is_enabled(adapter)) { + netif_warn(adapter, drv, adapter->netdev, + "PTP already disabled\n"); + goto done; + } + lan743x_csr_write(adapter, PTP_CMD_CTL, PTP_CMD_CTL_PTP_DISABLE_); + lan743x_ptp_wait_till_cmd_done(adapter, PTP_CMD_CTL_PTP_ENABLE_); +done: + mutex_unlock(&ptp->command_lock); +} + +static void lan743x_ptp_reset(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + mutex_lock(&ptp->command_lock); + + if (lan743x_ptp_is_enabled(adapter)) { + netif_err(adapter, drv, adapter->netdev, + "Attempting reset while enabled\n"); + goto done; + } + + lan743x_csr_write(adapter, PTP_CMD_CTL, PTP_CMD_CTL_PTP_RESET_); + lan743x_ptp_wait_till_cmd_done(adapter, PTP_CMD_CTL_PTP_RESET_); +done: + mutex_unlock(&ptp->command_lock); +} + +static void lan743x_ptp_clock_set(struct lan743x_adapter *adapter, + u32 seconds, u32 nano_seconds, + u32 sub_nano_seconds) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + mutex_lock(&ptp->command_lock); + + lan743x_csr_write(adapter, PTP_CLOCK_SEC, seconds); + lan743x_csr_write(adapter, PTP_CLOCK_NS, nano_seconds); + lan743x_csr_write(adapter, PTP_CLOCK_SUBNS, sub_nano_seconds); + + lan743x_csr_write(adapter, PTP_CMD_CTL, PTP_CMD_CTL_PTP_CLOCK_LOAD_); + lan743x_ptp_wait_till_cmd_done(adapter, PTP_CMD_CTL_PTP_CLOCK_LOAD_); + mutex_unlock(&ptp->command_lock); +} + +bool lan743x_ptp_request_tx_timestamp(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + bool result = false; + + spin_lock_bh(&ptp->tx_ts_lock); + if (ptp->pending_tx_timestamps < LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS) { + /* request granted */ + ptp->pending_tx_timestamps++; + result = true; + } + spin_unlock_bh(&ptp->tx_ts_lock); + return result; +} + +void lan743x_ptp_unrequest_tx_timestamp(struct lan743x_adapter *adapter) +{ + struct lan743x_ptp *ptp = &adapter->ptp; + + spin_lock_bh(&ptp->tx_ts_lock); + if (ptp->pending_tx_timestamps > 0) + ptp->pending_tx_timestamps--; + else + netif_err(adapter, drv, adapter->netdev, + "unrequest failed, pending_tx_timestamps==0\n"); + spin_unlock_bh(&ptp->tx_ts_lock); +} + +void lan743x_ptp_tx_timestamp_skb(struct lan743x_adapter *adapter, + struct sk_buff *skb, bool ignore_sync) +{ + lan743x_ptp_tx_ts_enqueue_skb(adapter, skb, ignore_sync); + + lan743x_ptp_tx_ts_complete(adapter); +} + +int lan743x_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + struct lan743x_adapter *adapter = netdev_priv(netdev); + struct hwtstamp_config config; + int ret = 0; + int index; + + if (!ifr) { + netif_err(adapter, drv, adapter->netdev, + "SIOCSHWTSTAMP, ifr == NULL\n"); + return -EINVAL; + } + + if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) + return -EFAULT; + + if (config.flags) { + netif_warn(adapter, drv, adapter->netdev, + "ignoring hwtstamp_config.flags == 0x%08X, expected 0\n", + config.flags); + } + + switch (config.tx_type) { + case HWTSTAMP_TX_OFF: + for (index = 0; index < LAN743X_MAX_TX_CHANNELS; + index++) + lan743x_tx_set_timestamping_mode(&adapter->tx[index], + false, false); + lan743x_ptp_set_sync_ts_insert(adapter, false); + break; + case HWTSTAMP_TX_ON: + for (index = 0; index < LAN743X_MAX_TX_CHANNELS; + index++) + lan743x_tx_set_timestamping_mode(&adapter->tx[index], + true, false); + lan743x_ptp_set_sync_ts_insert(adapter, false); + break; + case HWTSTAMP_TX_ONESTEP_SYNC: + for (index = 0; index < LAN743X_MAX_TX_CHANNELS; + index++) + lan743x_tx_set_timestamping_mode(&adapter->tx[index], + true, true); + + lan743x_ptp_set_sync_ts_insert(adapter, true); + break; + default: + netif_warn(adapter, drv, adapter->netdev, + " tx_type = %d, UNKNOWN\n", config.tx_type); + ret = -EINVAL; + break; + } + + if (!ret) + return copy_to_user(ifr->ifr_data, &config, + sizeof(config)) ? -EFAULT : 0; + return ret; +} diff --git a/drivers/net/ethernet/microchip/lan743x_ptp.h b/drivers/net/ethernet/microchip/lan743x_ptp.h new file mode 100644 index 000000000000..20f2223024dc --- /dev/null +++ b/drivers/net/ethernet/microchip/lan743x_ptp.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (C) 2018 Microchip Technology Inc. */ + +#ifndef _LAN743X_PTP_H +#define _LAN743X_PTP_H + +#include "linux/ptp_clock_kernel.h" +#include "linux/netdevice.h" + +struct lan743x_adapter; + +/* GPIO */ +struct lan743x_gpio { + /* gpio_lock: used to prevent concurrent access to gpio settings */ + spinlock_t gpio_lock; + + int used_bits; + int output_bits; + int ptp_bits; + u32 gpio_cfg0; + u32 gpio_cfg1; + u32 gpio_cfg2; + u32 gpio_cfg3; +}; + +int lan743x_gpio_init(struct lan743x_adapter *adapter); + +void lan743x_ptp_isr(void *context); +bool lan743x_ptp_request_tx_timestamp(struct lan743x_adapter *adapter); +void lan743x_ptp_unrequest_tx_timestamp(struct lan743x_adapter *adapter); +void lan743x_ptp_tx_timestamp_skb(struct lan743x_adapter *adapter, + struct sk_buff *skb, bool ignore_sync); +int lan743x_ptp_init(struct lan743x_adapter *adapter); +int lan743x_ptp_open(struct lan743x_adapter *adapter); +void lan743x_ptp_close(struct lan743x_adapter *adapter); +void lan743x_ptp_update_latency(struct lan743x_adapter *adapter, + u32 link_speed); + +int lan743x_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd); + +#define LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS (4) + +#define PTP_FLAG_PTP_CLOCK_REGISTERED BIT(1) +#define PTP_FLAG_ISR_ENABLED BIT(2) + +struct lan743x_ptp { + int flags; + + /* command_lock: used to prevent concurrent ptp commands */ + struct mutex command_lock; + +#ifdef CONFIG_PTP_1588_CLOCK + struct ptp_clock *ptp_clock; + struct ptp_clock_info ptp_clock_info; + struct ptp_pin_desc pin_config[1]; +#endif /* CONFIG_PTP_1588_CLOCK */ + +#define LAN743X_PTP_NUMBER_OF_EVENT_CHANNELS (2) + unsigned long used_event_ch; + + int perout_event_ch; + int perout_gpio_bit; + + /* tx_ts_lock: used to prevent concurrent access to timestamp arrays */ + spinlock_t tx_ts_lock; + int pending_tx_timestamps; + struct sk_buff *tx_ts_skb_queue[LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS]; + unsigned int tx_ts_ignore_sync_queue; + int tx_ts_skb_queue_size; + u32 tx_ts_seconds_queue[LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS]; + u32 tx_ts_nseconds_queue[LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS]; + u32 tx_ts_header_queue[LAN743X_PTP_NUMBER_OF_TX_TIMESTAMPS]; + int tx_ts_queue_size; +}; + +#endif /* _LAN743X_PTP_H */ diff --git a/drivers/net/ethernet/neterion/vxge/vxge-config.c b/drivers/net/ethernet/neterion/vxge/vxge-config.c index ae8149831e47..398011c87643 100644 --- a/drivers/net/ethernet/neterion/vxge/vxge-config.c +++ b/drivers/net/ethernet/neterion/vxge/vxge-config.c @@ -1094,12 +1094,9 @@ static void __vxge_hw_blockpool_destroy(struct __vxge_hw_blockpool *blockpool) { struct __vxge_hw_device *hldev; struct list_head *p, *n; - u16 ret; - if (blockpool == NULL) { - ret = 1; - goto exit; - } + if (!blockpool) + return; hldev = blockpool->hldev; @@ -1122,8 +1119,7 @@ static void __vxge_hw_blockpool_destroy(struct __vxge_hw_blockpool *blockpool) list_del(&((struct __vxge_hw_blockpool_entry *)p)->item); kfree((void *)p); } - ret = 0; -exit: + return; } @@ -2259,14 +2255,11 @@ static void vxge_hw_blockpool_block_add(struct __vxge_hw_device *devh, struct __vxge_hw_blockpool *blockpool; struct __vxge_hw_blockpool_entry *entry = NULL; dma_addr_t dma_addr; - enum vxge_hw_status status = VXGE_HW_OK; - u32 req_out; blockpool = &devh->block_pool; if (block_addr == NULL) { blockpool->req_out--; - status = VXGE_HW_FAIL; goto exit; } @@ -2276,7 +2269,6 @@ static void vxge_hw_blockpool_block_add(struct __vxge_hw_device *devh, if (unlikely(pci_dma_mapping_error(devh->pdev, dma_addr))) { vxge_os_dma_free(devh->pdev, block_addr, &acc_handle); blockpool->req_out--; - status = VXGE_HW_FAIL; goto exit; } @@ -2291,7 +2283,7 @@ static void vxge_hw_blockpool_block_add(struct __vxge_hw_device *devh, else list_del(&entry->item); - if (entry != NULL) { + if (entry) { entry->length = length; entry->memblock = block_addr; entry->dma_addr = dma_addr; @@ -2299,13 +2291,10 @@ static void vxge_hw_blockpool_block_add(struct __vxge_hw_device *devh, entry->dma_handle = dma_h; list_add(&entry->item, &blockpool->free_block_list); blockpool->pool_size++; - status = VXGE_HW_OK; - } else - status = VXGE_HW_ERR_OUT_OF_MEMORY; + } blockpool->req_out--; - req_out = blockpool->req_out; exit: return; } @@ -2357,7 +2346,6 @@ static void *__vxge_hw_blockpool_malloc(struct __vxge_hw_device *devh, u32 size, struct __vxge_hw_blockpool_entry *entry = NULL; struct __vxge_hw_blockpool *blockpool; void *memblock = NULL; - enum vxge_hw_status status = VXGE_HW_OK; blockpool = &devh->block_pool; @@ -2367,10 +2355,8 @@ static void *__vxge_hw_blockpool_malloc(struct __vxge_hw_device *devh, u32 size, &dma_object->handle, &dma_object->acc_handle); - if (memblock == NULL) { - status = VXGE_HW_ERR_OUT_OF_MEMORY; + if (!memblock) goto exit; - } dma_object->addr = pci_map_single(devh->pdev, memblock, size, PCI_DMA_BIDIRECTIONAL); @@ -2379,7 +2365,6 @@ static void *__vxge_hw_blockpool_malloc(struct __vxge_hw_device *devh, u32 size, dma_object->addr))) { vxge_os_dma_free(devh->pdev, memblock, &dma_object->acc_handle); - status = VXGE_HW_ERR_OUT_OF_MEMORY; goto exit; } diff --git a/drivers/net/ethernet/ni/nixge.c b/drivers/net/ethernet/ni/nixge.c index 9ae4fcef725e..76efed058f33 100644 --- a/drivers/net/ethernet/ni/nixge.c +++ b/drivers/net/ethernet/ni/nixge.c @@ -503,7 +503,6 @@ static int nixge_start_xmit(struct sk_buff *skb, struct net_device *ndev) tx_skb->skb = skb; cur_p->cntrl |= XAXIDMA_BD_CTRL_TXEOF_MASK; - cur_p->app4 = (unsigned long)skb; tail_p = priv->tx_bd_p + sizeof(*priv->tx_bd_v) * priv->tx_bd_tail; /* Start the transfer */ @@ -739,22 +738,12 @@ static void nixge_dma_err_handler(unsigned long data) cur_p->phys = 0; cur_p->cntrl = 0; cur_p->status = 0; - cur_p->app0 = 0; - cur_p->app1 = 0; - cur_p->app2 = 0; - cur_p->app3 = 0; - cur_p->app4 = 0; cur_p->sw_id_offset = 0; } for (i = 0; i < RX_BD_NUM; i++) { cur_p = &lp->rx_bd_v[i]; cur_p->status = 0; - cur_p->app0 = 0; - cur_p->app1 = 0; - cur_p->app2 = 0; - cur_p->app3 = 0; - cur_p->app4 = 0; } lp->tx_bd_ci = 0; diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 4b5d98f60d32..46d0f2eaa0c0 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -536,7 +536,7 @@ static int qede_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) return 0; } -int qede_setup_tc(struct net_device *ndev, u8 num_tc) +static int qede_setup_tc(struct net_device *ndev, u8 num_tc) { struct qede_dev *edev = netdev_priv(ndev); int cos, count, offset; diff --git a/drivers/net/ethernet/realtek/r8169.c b/drivers/net/ethernet/realtek/r8169.c index 14d185b10cda..0d9c3831838f 100644 --- a/drivers/net/ethernet/realtek/r8169.c +++ b/drivers/net/ethernet/realtek/r8169.c @@ -7076,6 +7076,11 @@ static int rtl_alloc_irq(struct rtl8169_private *tp) RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~MSIEnable); RTL_W8(tp, Cfg9346, Cfg9346_Lock); flags = PCI_IRQ_LEGACY; + } else if (tp->mac_version == RTL_GIGA_MAC_VER_40) { + /* This version was reported to have issues with resume + * from suspend when using MSI-X + */ + flags = PCI_IRQ_LEGACY | PCI_IRQ_MSI; } else { flags = PCI_IRQ_ALL_TYPES; } diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index 01589b6982e4..7aa5ebb6766c 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -232,8 +232,7 @@ #define NETSEC_EEPROM_PKT_ME_ADDRESS 0x20 #define NETSEC_EEPROM_PKT_ME_SIZE 0x24 -#define DESC_NUM 128 -#define NAPI_BUDGET (DESC_NUM / 2) +#define DESC_NUM 256 #define DESC_SZ sizeof(struct netsec_de) @@ -642,8 +641,6 @@ static struct sk_buff *netsec_get_rx_pkt_data(struct netsec_priv *priv, tmp_skb = netsec_alloc_skb(priv, &td); - dma_rmb(); - tail = dring->tail; if (!tmp_skb) { @@ -657,8 +654,6 @@ static struct sk_buff *netsec_get_rx_pkt_data(struct netsec_priv *priv, /* move tail ahead */ dring->tail = (dring->tail + 1) % DESC_NUM; - dring->pkt_cnt--; - return skb; } @@ -731,25 +726,24 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget) struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX]; struct net_device *ndev = priv->ndev; struct netsec_rx_pkt_info rx_info; - int done = 0, rx_num = 0; + int done = 0; struct netsec_desc desc; struct sk_buff *skb; u16 len; while (done < budget) { - if (!rx_num) { - rx_num = netsec_read(priv, NETSEC_REG_NRM_RX_PKTCNT); - dring->pkt_cnt += rx_num; + u16 idx = dring->tail; + struct netsec_de *de = dring->vaddr + (DESC_SZ * idx); - /* move head 'rx_num' */ - dring->head = (dring->head + rx_num) % DESC_NUM; + if (de->attr & (1U << NETSEC_RX_PKT_OWN_FIELD)) + break; - rx_num = dring->pkt_cnt; - if (!rx_num) - break; - } + /* This barrier is needed to keep us from reading + * any other fields out of the netsec_de until we have + * verified the descriptor has been written back + */ + dma_rmb(); done++; - rx_num--; skb = netsec_get_rx_pkt_data(priv, &rx_info, &desc, &len); if (unlikely(!skb) || rx_info.err_flag) { netif_err(priv, drv, priv->ndev, @@ -1664,7 +1658,7 @@ static int netsec_probe(struct platform_device *pdev) dev_info(&pdev->dev, "hardware revision %d.%d\n", hw_ver >> 16, hw_ver & 0xffff); - netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_BUDGET); + netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_POLL_WEIGHT); ndev->netdev_ops = &netsec_netdev_ops; ndev->ethtool_ops = &netsec_ethtool_ops; diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index f051ce35a440..832bce07c385 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -2358,14 +2358,16 @@ static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev, int i; for (i = 0; i < cpsw->data.slaves; i++) { - if (vid == cpsw->slaves[i].port_vlan) - return -EINVAL; + if (vid == cpsw->slaves[i].port_vlan) { + ret = -EINVAL; + goto err; + } } } dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid); ret = cpsw_add_vlan_ale_entry(priv, vid); - +err: pm_runtime_put(cpsw->dev); return ret; } @@ -2391,22 +2393,17 @@ static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev, for (i = 0; i < cpsw->data.slaves; i++) { if (vid == cpsw->slaves[i].port_vlan) - return -EINVAL; + goto err; } } dev_info(priv->dev, "removing vlanid %d from vlan filter\n", vid); ret = cpsw_ale_del_vlan(cpsw->ale, vid, 0); - if (ret != 0) - return ret; - - ret = cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, - HOST_PORT_NUM, ALE_VLAN, vid); - if (ret != 0) - return ret; - - ret = cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast, - 0, ALE_VLAN, vid); + ret |= cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, + HOST_PORT_NUM, ALE_VLAN, vid); + ret |= cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast, + 0, ALE_VLAN, vid); +err: pm_runtime_put(cpsw->dev); return ret; } diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c index 93dc05c194d3..5766225a4ce1 100644 --- a/drivers/net/ethernet/ti/cpsw_ale.c +++ b/drivers/net/ethernet/ti/cpsw_ale.c @@ -394,7 +394,7 @@ int cpsw_ale_del_mcast(struct cpsw_ale *ale, u8 *addr, int port_mask, idx = cpsw_ale_match_addr(ale, addr, (flags & ALE_VLAN) ? vid : 0); if (idx < 0) - return -EINVAL; + return -ENOENT; cpsw_ale_read(ale, idx, ale_entry); diff --git a/drivers/net/phy/dp83640.c b/drivers/net/phy/dp83640.c index 79e9b103188b..29aa8d772b0c 100644 --- a/drivers/net/phy/dp83640.c +++ b/drivers/net/phy/dp83640.c @@ -757,13 +757,16 @@ static int decode_evnt(struct dp83640_private *dp83640, phy_txts = data; - switch (words) { /* fall through in every case */ + switch (words) { case 3: dp83640->edata.sec_hi = phy_txts->sec_hi; + /* fall through */ case 2: dp83640->edata.sec_lo = phy_txts->sec_lo; + /* fall through */ case 1: dp83640->edata.ns_hi = phy_txts->ns_hi; + /* fall through */ case 0: dp83640->edata.ns_lo = phy_txts->ns_lo; } diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 124211afb023..97742708460b 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -4415,7 +4415,6 @@ out1: static int rtl8152_system_suspend(struct r8152 *tp) { struct net_device *netdev = tp->netdev; - int ret = 0; netif_device_detach(netdev); @@ -4430,7 +4429,7 @@ static int rtl8152_system_suspend(struct r8152 *tp) napi_enable(napi); } - return ret; + return 0; } static int rtl8152_suspend(struct usb_interface *intf, pm_message_t message) diff --git a/drivers/net/veth.c b/drivers/net/veth.c index a69ad39ee57e..e3202af72df5 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -17,22 +17,47 @@ #include <net/rtnetlink.h> #include <net/dst.h> #include <net/xfrm.h> +#include <net/xdp.h> #include <linux/veth.h> #include <linux/module.h> +#include <linux/bpf.h> +#include <linux/filter.h> +#include <linux/ptr_ring.h> +#include <linux/bpf_trace.h> #define DRV_NAME "veth" #define DRV_VERSION "1.0" +#define VETH_XDP_FLAG BIT(0) +#define VETH_RING_SIZE 256 +#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) + +/* Separating two types of XDP xmit */ +#define VETH_XDP_TX BIT(0) +#define VETH_XDP_REDIR BIT(1) + struct pcpu_vstats { u64 packets; u64 bytes; struct u64_stats_sync syncp; }; +struct veth_rq { + struct napi_struct xdp_napi; + struct net_device *dev; + struct bpf_prog __rcu *xdp_prog; + struct xdp_mem_info xdp_mem; + bool rx_notify_masked; + struct ptr_ring xdp_ring; + struct xdp_rxq_info xdp_rxq; +}; + struct veth_priv { struct net_device __rcu *peer; atomic64_t dropped; - unsigned requested_headroom; + struct bpf_prog *_xdp_prog; + struct veth_rq *rq; + unsigned int requested_headroom; }; /* @@ -98,11 +123,67 @@ static const struct ethtool_ops veth_ethtool_ops = { .get_link_ksettings = veth_get_link_ksettings, }; +/* general routines */ + +static bool veth_is_xdp_frame(void *ptr) +{ + return (unsigned long)ptr & VETH_XDP_FLAG; +} + +static void *veth_ptr_to_xdp(void *ptr) +{ + return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); +} + +static void *veth_xdp_to_ptr(void *ptr) +{ + return (void *)((unsigned long)ptr | VETH_XDP_FLAG); +} + +static void veth_ptr_free(void *ptr) +{ + if (veth_is_xdp_frame(ptr)) + xdp_return_frame(veth_ptr_to_xdp(ptr)); + else + kfree_skb(ptr); +} + +static void __veth_xdp_flush(struct veth_rq *rq) +{ + /* Write ptr_ring before reading rx_notify_masked */ + smp_mb(); + if (!rq->rx_notify_masked) { + rq->rx_notify_masked = true; + napi_schedule(&rq->xdp_napi); + } +} + +static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) +{ + if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) { + dev_kfree_skb_any(skb); + return NET_RX_DROP; + } + + return NET_RX_SUCCESS; +} + +static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, + struct veth_rq *rq, bool xdp) +{ + return __dev_forward_skb(dev, skb) ?: xdp ? + veth_xdp_rx(rq, skb) : + netif_rx(skb); +} + static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) { - struct veth_priv *priv = netdev_priv(dev); + struct veth_priv *rcv_priv, *priv = netdev_priv(dev); + struct veth_rq *rq = NULL; struct net_device *rcv; int length = skb->len; + bool rcv_xdp = false; + int rxq; rcu_read_lock(); rcv = rcu_dereference(priv->peer); @@ -111,7 +192,16 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) goto drop; } - if (likely(dev_forward_skb(rcv, skb) == NET_RX_SUCCESS)) { + rcv_priv = netdev_priv(rcv); + rxq = skb_get_queue_mapping(skb); + if (rxq < rcv->real_num_rx_queues) { + rq = &rcv_priv->rq[rxq]; + rcv_xdp = rcu_access_pointer(rq->xdp_prog); + if (rcv_xdp) + skb_record_rx_queue(skb, rxq); + } + + if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) { struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats); u64_stats_update_begin(&stats->syncp); @@ -122,14 +212,15 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) drop: atomic64_inc(&priv->dropped); } + + if (rcv_xdp) + __veth_xdp_flush(rq); + rcu_read_unlock(); + return NETDEV_TX_OK; } -/* - * general routines - */ - static u64 veth_stats_one(struct pcpu_vstats *result, struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); @@ -179,18 +270,502 @@ static void veth_set_multicast_list(struct net_device *dev) { } +static struct sk_buff *veth_build_skb(void *head, int headroom, int len, + int buflen) +{ + struct sk_buff *skb; + + if (!buflen) { + buflen = SKB_DATA_ALIGN(headroom + len) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + } + skb = build_skb(head, buflen); + if (!skb) + return NULL; + + skb_reserve(skb, headroom); + skb_put(skb, len); + + return skb; +} + +static int veth_select_rxq(struct net_device *dev) +{ + return smp_processor_id() % dev->real_num_rx_queues; +} + +static int veth_xdp_xmit(struct net_device *dev, int n, + struct xdp_frame **frames, u32 flags) +{ + struct veth_priv *rcv_priv, *priv = netdev_priv(dev); + struct net_device *rcv; + unsigned int max_len; + struct veth_rq *rq; + int i, drops = 0; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + rcv = rcu_dereference(priv->peer); + if (unlikely(!rcv)) + return -ENXIO; + + rcv_priv = netdev_priv(rcv); + rq = &rcv_priv->rq[veth_select_rxq(rcv)]; + /* Non-NULL xdp_prog ensures that xdp_ring is initialized on receive + * side. This means an XDP program is loaded on the peer and the peer + * device is up. + */ + if (!rcu_access_pointer(rq->xdp_prog)) + return -ENXIO; + + max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; + + spin_lock(&rq->xdp_ring.producer_lock); + for (i = 0; i < n; i++) { + struct xdp_frame *frame = frames[i]; + void *ptr = veth_xdp_to_ptr(frame); + + if (unlikely(frame->len > max_len || + __ptr_ring_produce(&rq->xdp_ring, ptr))) { + xdp_return_frame_rx_napi(frame); + drops++; + } + } + spin_unlock(&rq->xdp_ring.producer_lock); + + if (flags & XDP_XMIT_FLUSH) + __veth_xdp_flush(rq); + + return n - drops; +} + +static void veth_xdp_flush(struct net_device *dev) +{ + struct veth_priv *rcv_priv, *priv = netdev_priv(dev); + struct net_device *rcv; + struct veth_rq *rq; + + rcu_read_lock(); + rcv = rcu_dereference(priv->peer); + if (unlikely(!rcv)) + goto out; + + rcv_priv = netdev_priv(rcv); + rq = &rcv_priv->rq[veth_select_rxq(rcv)]; + /* xdp_ring is initialized on receive side? */ + if (unlikely(!rcu_access_pointer(rq->xdp_prog))) + goto out; + + __veth_xdp_flush(rq); +out: + rcu_read_unlock(); +} + +static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp) +{ + struct xdp_frame *frame = convert_to_xdp_frame(xdp); + + if (unlikely(!frame)) + return -EOVERFLOW; + + return veth_xdp_xmit(dev, 1, &frame, 0); +} + +static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, + struct xdp_frame *frame, + unsigned int *xdp_xmit) +{ + void *hard_start = frame->data - frame->headroom; + void *head = hard_start - sizeof(struct xdp_frame); + int len = frame->len, delta = 0; + struct xdp_frame orig_frame; + struct bpf_prog *xdp_prog; + unsigned int headroom; + struct sk_buff *skb; + + rcu_read_lock(); + xdp_prog = rcu_dereference(rq->xdp_prog); + if (likely(xdp_prog)) { + struct xdp_buff xdp; + u32 act; + + xdp.data_hard_start = hard_start; + xdp.data = frame->data; + xdp.data_end = frame->data + frame->len; + xdp.data_meta = frame->data - frame->metasize; + xdp.rxq = &rq->xdp_rxq; + + act = bpf_prog_run_xdp(xdp_prog, &xdp); + + switch (act) { + case XDP_PASS: + delta = frame->data - xdp.data; + len = xdp.data_end - xdp.data; + break; + case XDP_TX: + orig_frame = *frame; + xdp.data_hard_start = head; + xdp.rxq->mem = frame->mem; + if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) { + trace_xdp_exception(rq->dev, xdp_prog, act); + frame = &orig_frame; + goto err_xdp; + } + *xdp_xmit |= VETH_XDP_TX; + rcu_read_unlock(); + goto xdp_xmit; + case XDP_REDIRECT: + orig_frame = *frame; + xdp.data_hard_start = head; + xdp.rxq->mem = frame->mem; + if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { + frame = &orig_frame; + goto err_xdp; + } + *xdp_xmit |= VETH_XDP_REDIR; + rcu_read_unlock(); + goto xdp_xmit; + default: + bpf_warn_invalid_xdp_action(act); + case XDP_ABORTED: + trace_xdp_exception(rq->dev, xdp_prog, act); + case XDP_DROP: + goto err_xdp; + } + } + rcu_read_unlock(); + + headroom = sizeof(struct xdp_frame) + frame->headroom - delta; + skb = veth_build_skb(head, headroom, len, 0); + if (!skb) { + xdp_return_frame(frame); + goto err; + } + + xdp_scrub_frame(frame); + skb->protocol = eth_type_trans(skb, rq->dev); +err: + return skb; +err_xdp: + rcu_read_unlock(); + xdp_return_frame(frame); +xdp_xmit: + return NULL; +} + +static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, struct sk_buff *skb, + unsigned int *xdp_xmit) +{ + u32 pktlen, headroom, act, metalen; + void *orig_data, *orig_data_end; + struct bpf_prog *xdp_prog; + int mac_len, delta, off; + struct xdp_buff xdp; + + rcu_read_lock(); + xdp_prog = rcu_dereference(rq->xdp_prog); + if (unlikely(!xdp_prog)) { + rcu_read_unlock(); + goto out; + } + + mac_len = skb->data - skb_mac_header(skb); + pktlen = skb->len + mac_len; + headroom = skb_headroom(skb) - mac_len; + + if (skb_shared(skb) || skb_head_is_locked(skb) || + skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) { + struct sk_buff *nskb; + int size, head_off; + void *head, *start; + struct page *page; + + size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + if (size > PAGE_SIZE) + goto drop; + + page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); + if (!page) + goto drop; + + head = page_address(page); + start = head + VETH_XDP_HEADROOM; + if (skb_copy_bits(skb, -mac_len, start, pktlen)) { + page_frag_free(head); + goto drop; + } + + nskb = veth_build_skb(head, + VETH_XDP_HEADROOM + mac_len, skb->len, + PAGE_SIZE); + if (!nskb) { + page_frag_free(head); + goto drop; + } + + skb_copy_header(nskb, skb); + head_off = skb_headroom(nskb) - skb_headroom(skb); + skb_headers_offset_update(nskb, head_off); + if (skb->sk) + skb_set_owner_w(nskb, skb->sk); + consume_skb(skb); + skb = nskb; + } + + xdp.data_hard_start = skb->head; + xdp.data = skb_mac_header(skb); + xdp.data_end = xdp.data + pktlen; + xdp.data_meta = xdp.data; + xdp.rxq = &rq->xdp_rxq; + orig_data = xdp.data; + orig_data_end = xdp.data_end; + + act = bpf_prog_run_xdp(xdp_prog, &xdp); + + switch (act) { + case XDP_PASS: + break; + case XDP_TX: + get_page(virt_to_page(xdp.data)); + consume_skb(skb); + xdp.rxq->mem = rq->xdp_mem; + if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) { + trace_xdp_exception(rq->dev, xdp_prog, act); + goto err_xdp; + } + *xdp_xmit |= VETH_XDP_TX; + rcu_read_unlock(); + goto xdp_xmit; + case XDP_REDIRECT: + get_page(virt_to_page(xdp.data)); + consume_skb(skb); + xdp.rxq->mem = rq->xdp_mem; + if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) + goto err_xdp; + *xdp_xmit |= VETH_XDP_REDIR; + rcu_read_unlock(); + goto xdp_xmit; + default: + bpf_warn_invalid_xdp_action(act); + case XDP_ABORTED: + trace_xdp_exception(rq->dev, xdp_prog, act); + case XDP_DROP: + goto drop; + } + rcu_read_unlock(); + + delta = orig_data - xdp.data; + off = mac_len + delta; + if (off > 0) + __skb_push(skb, off); + else if (off < 0) + __skb_pull(skb, -off); + skb->mac_header -= delta; + off = xdp.data_end - orig_data_end; + if (off != 0) + __skb_put(skb, off); + skb->protocol = eth_type_trans(skb, rq->dev); + + metalen = xdp.data - xdp.data_meta; + if (metalen) + skb_metadata_set(skb, metalen); +out: + return skb; +drop: + rcu_read_unlock(); + kfree_skb(skb); + return NULL; +err_xdp: + rcu_read_unlock(); + page_frag_free(xdp.data); +xdp_xmit: + return NULL; +} + +static int veth_xdp_rcv(struct veth_rq *rq, int budget, unsigned int *xdp_xmit) +{ + int i, done = 0; + + for (i = 0; i < budget; i++) { + void *ptr = __ptr_ring_consume(&rq->xdp_ring); + struct sk_buff *skb; + + if (!ptr) + break; + + if (veth_is_xdp_frame(ptr)) { + skb = veth_xdp_rcv_one(rq, veth_ptr_to_xdp(ptr), + xdp_xmit); + } else { + skb = veth_xdp_rcv_skb(rq, ptr, xdp_xmit); + } + + if (skb) + napi_gro_receive(&rq->xdp_napi, skb); + + done++; + } + + return done; +} + +static int veth_poll(struct napi_struct *napi, int budget) +{ + struct veth_rq *rq = + container_of(napi, struct veth_rq, xdp_napi); + unsigned int xdp_xmit = 0; + int done; + + xdp_set_return_frame_no_direct(); + done = veth_xdp_rcv(rq, budget, &xdp_xmit); + + if (done < budget && napi_complete_done(napi, done)) { + /* Write rx_notify_masked before reading ptr_ring */ + smp_store_mb(rq->rx_notify_masked, false); + if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { + rq->rx_notify_masked = true; + napi_schedule(&rq->xdp_napi); + } + } + + if (xdp_xmit & VETH_XDP_TX) + veth_xdp_flush(rq->dev); + if (xdp_xmit & VETH_XDP_REDIR) + xdp_do_flush_map(); + xdp_clear_return_frame_no_direct(); + + return done; +} + +static int veth_napi_add(struct net_device *dev) +{ + struct veth_priv *priv = netdev_priv(dev); + int err, i; + + for (i = 0; i < dev->real_num_rx_queues; i++) { + struct veth_rq *rq = &priv->rq[i]; + + err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); + if (err) + goto err_xdp_ring; + } + + for (i = 0; i < dev->real_num_rx_queues; i++) { + struct veth_rq *rq = &priv->rq[i]; + + netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); + napi_enable(&rq->xdp_napi); + } + + return 0; +err_xdp_ring: + for (i--; i >= 0; i--) + ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); + + return err; +} + +static void veth_napi_del(struct net_device *dev) +{ + struct veth_priv *priv = netdev_priv(dev); + int i; + + for (i = 0; i < dev->real_num_rx_queues; i++) { + struct veth_rq *rq = &priv->rq[i]; + + napi_disable(&rq->xdp_napi); + napi_hash_del(&rq->xdp_napi); + } + synchronize_net(); + + for (i = 0; i < dev->real_num_rx_queues; i++) { + struct veth_rq *rq = &priv->rq[i]; + + netif_napi_del(&rq->xdp_napi); + rq->rx_notify_masked = false; + ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); + } +} + +static int veth_enable_xdp(struct net_device *dev) +{ + struct veth_priv *priv = netdev_priv(dev); + int err, i; + + if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { + for (i = 0; i < dev->real_num_rx_queues; i++) { + struct veth_rq *rq = &priv->rq[i]; + + err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); + if (err < 0) + goto err_rxq_reg; + + err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, + MEM_TYPE_PAGE_SHARED, + NULL); + if (err < 0) + goto err_reg_mem; + + /* Save original mem info as it can be overwritten */ + rq->xdp_mem = rq->xdp_rxq.mem; + } + + err = veth_napi_add(dev); + if (err) + goto err_rxq_reg; + } + + for (i = 0; i < dev->real_num_rx_queues; i++) + rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); + + return 0; +err_reg_mem: + xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); +err_rxq_reg: + for (i--; i >= 0; i--) + xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); + + return err; +} + +static void veth_disable_xdp(struct net_device *dev) +{ + struct veth_priv *priv = netdev_priv(dev); + int i; + + for (i = 0; i < dev->real_num_rx_queues; i++) + rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); + veth_napi_del(dev); + for (i = 0; i < dev->real_num_rx_queues; i++) { + struct veth_rq *rq = &priv->rq[i]; + + rq->xdp_rxq.mem = rq->xdp_mem; + xdp_rxq_info_unreg(&rq->xdp_rxq); + } +} + static int veth_open(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); struct net_device *peer = rtnl_dereference(priv->peer); + int err; if (!peer) return -ENOTCONN; + if (priv->_xdp_prog) { + err = veth_enable_xdp(dev); + if (err) + return err; + } + if (peer->flags & IFF_UP) { netif_carrier_on(dev); netif_carrier_on(peer); } + return 0; } @@ -203,6 +778,9 @@ static int veth_close(struct net_device *dev) if (peer) netif_carrier_off(peer); + if (priv->_xdp_prog) + veth_disable_xdp(dev); + return 0; } @@ -228,7 +806,7 @@ static void veth_dev_free(struct net_device *dev) static void veth_poll_controller(struct net_device *dev) { /* veth only receives frames when its peer sends one - * Since it's a synchronous operation, we are guaranteed + * Since it has nothing to do with disabling irqs, we are guaranteed * never to have pending data when we poll for it so * there is nothing to do here. * @@ -253,6 +831,23 @@ static int veth_get_iflink(const struct net_device *dev) return iflink; } +static netdev_features_t veth_fix_features(struct net_device *dev, + netdev_features_t features) +{ + struct veth_priv *priv = netdev_priv(dev); + struct net_device *peer; + + peer = rtnl_dereference(priv->peer); + if (peer) { + struct veth_priv *peer_priv = netdev_priv(peer); + + if (peer_priv->_xdp_prog) + features &= ~NETIF_F_GSO_SOFTWARE; + } + + return features; +} + static void veth_set_rx_headroom(struct net_device *dev, int new_hr) { struct veth_priv *peer_priv, *priv = netdev_priv(dev); @@ -276,6 +871,103 @@ out: rcu_read_unlock(); } +static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct veth_priv *priv = netdev_priv(dev); + struct bpf_prog *old_prog; + struct net_device *peer; + unsigned int max_mtu; + int err; + + old_prog = priv->_xdp_prog; + priv->_xdp_prog = prog; + peer = rtnl_dereference(priv->peer); + + if (prog) { + if (!peer) { + NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); + err = -ENOTCONN; + goto err; + } + + max_mtu = PAGE_SIZE - VETH_XDP_HEADROOM - + peer->hard_header_len - + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + if (peer->mtu > max_mtu) { + NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); + err = -ERANGE; + goto err; + } + + if (dev->real_num_rx_queues < peer->real_num_tx_queues) { + NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); + err = -ENOSPC; + goto err; + } + + if (dev->flags & IFF_UP) { + err = veth_enable_xdp(dev); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); + goto err; + } + } + + if (!old_prog) { + peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; + peer->max_mtu = max_mtu; + } + } + + if (old_prog) { + if (!prog) { + if (dev->flags & IFF_UP) + veth_disable_xdp(dev); + + if (peer) { + peer->hw_features |= NETIF_F_GSO_SOFTWARE; + peer->max_mtu = ETH_MAX_MTU; + } + } + bpf_prog_put(old_prog); + } + + if ((!!old_prog ^ !!prog) && peer) + netdev_update_features(peer); + + return 0; +err: + priv->_xdp_prog = old_prog; + + return err; +} + +static u32 veth_xdp_query(struct net_device *dev) +{ + struct veth_priv *priv = netdev_priv(dev); + const struct bpf_prog *xdp_prog; + + xdp_prog = priv->_xdp_prog; + if (xdp_prog) + return xdp_prog->aux->id; + + return 0; +} + +static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + switch (xdp->command) { + case XDP_SETUP_PROG: + return veth_xdp_set(dev, xdp->prog, xdp->extack); + case XDP_QUERY_PROG: + xdp->prog_id = veth_xdp_query(dev); + return 0; + default: + return -EINVAL; + } +} + static const struct net_device_ops veth_netdev_ops = { .ndo_init = veth_dev_init, .ndo_open = veth_open, @@ -288,8 +980,11 @@ static const struct net_device_ops veth_netdev_ops = { .ndo_poll_controller = veth_poll_controller, #endif .ndo_get_iflink = veth_get_iflink, + .ndo_fix_features = veth_fix_features, .ndo_features_check = passthru_features_check, .ndo_set_rx_headroom = veth_set_rx_headroom, + .ndo_bpf = veth_xdp, + .ndo_xdp_xmit = veth_xdp_xmit, }; #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ @@ -345,13 +1040,31 @@ static int veth_validate(struct nlattr *tb[], struct nlattr *data[], return 0; } +static int veth_alloc_queues(struct net_device *dev) +{ + struct veth_priv *priv = netdev_priv(dev); + + priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL); + if (!priv->rq) + return -ENOMEM; + + return 0; +} + +static void veth_free_queues(struct net_device *dev) +{ + struct veth_priv *priv = netdev_priv(dev); + + kfree(priv->rq); +} + static struct rtnl_link_ops veth_link_ops; static int veth_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) { - int err; + int err, i; struct net_device *peer; struct veth_priv *priv; char ifname[IFNAMSIZ]; @@ -404,6 +1117,12 @@ static int veth_newlink(struct net *src_net, struct net_device *dev, return PTR_ERR(peer); } + err = veth_alloc_queues(peer); + if (err) { + put_net(net); + goto err_peer_alloc_queues; + } + if (!ifmp || !tbp[IFLA_ADDRESS]) eth_hw_addr_random(peer); @@ -432,6 +1151,10 @@ static int veth_newlink(struct net *src_net, struct net_device *dev, * should be re-allocated */ + err = veth_alloc_queues(dev); + if (err) + goto err_alloc_queues; + if (tb[IFLA_ADDRESS] == NULL) eth_hw_addr_random(dev); @@ -451,19 +1174,28 @@ static int veth_newlink(struct net *src_net, struct net_device *dev, */ priv = netdev_priv(dev); + for (i = 0; i < dev->real_num_rx_queues; i++) + priv->rq[i].dev = dev; rcu_assign_pointer(priv->peer, peer); priv = netdev_priv(peer); + for (i = 0; i < peer->real_num_rx_queues; i++) + priv->rq[i].dev = peer; rcu_assign_pointer(priv->peer, dev); + return 0; err_register_dev: + veth_free_queues(dev); +err_alloc_queues: /* nothing to do */ err_configure_peer: unregister_netdevice(peer); return err; err_register_peer: + veth_free_queues(peer); +err_peer_alloc_queues: free_netdev(peer); return err; } diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 39a7f4452587..765920905226 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -30,7 +30,7 @@ #include <linux/cpu.h> #include <linux/average.h> #include <linux/filter.h> -#include <linux/netdevice.h> +#include <linux/kernel.h> #include <linux/pci.h> #include <net/route.h> #include <net/xdp.h> @@ -1878,8 +1878,8 @@ static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu) if (vi->affinity_hint_set) { for (i = 0; i < vi->max_queue_pairs; i++) { - virtqueue_set_affinity(vi->rq[i].vq, -1); - virtqueue_set_affinity(vi->sq[i].vq, -1); + virtqueue_set_affinity(vi->rq[i].vq, NULL); + virtqueue_set_affinity(vi->sq[i].vq, NULL); } vi->affinity_hint_set = false; @@ -1888,30 +1888,41 @@ static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu) static void virtnet_set_affinity(struct virtnet_info *vi) { - int i; - int cpu; + cpumask_var_t mask; + int stragglers; + int group_size; + int i, j, cpu; + int num_cpu; + int stride; - /* In multiqueue mode, when the number of cpu is equal to the number of - * queue pairs, we let the queue pairs to be private to one cpu by - * setting the affinity hint to eliminate the contention. - */ - if (vi->curr_queue_pairs == 1 || - vi->max_queue_pairs != num_online_cpus()) { + if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { virtnet_clean_affinity(vi, -1); return; } - i = 0; - for_each_online_cpu(cpu) { - const unsigned long *mask = cpumask_bits(cpumask_of(cpu)); + num_cpu = num_online_cpus(); + stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); + stragglers = num_cpu >= vi->curr_queue_pairs ? + num_cpu % vi->curr_queue_pairs : + 0; + cpu = cpumask_next(-1, cpu_online_mask); - virtqueue_set_affinity(vi->rq[i].vq, cpu); - virtqueue_set_affinity(vi->sq[i].vq, cpu); - __netif_set_xps_queue(vi->dev, mask, i, false); - i++; + for (i = 0; i < vi->curr_queue_pairs; i++) { + group_size = stride + (i < stragglers ? 1 : 0); + + for (j = 0; j < group_size; j++) { + cpumask_set_cpu(cpu, mask); + cpu = cpumask_next_wrap(cpu, cpu_online_mask, + nr_cpu_ids, false); + } + virtqueue_set_affinity(vi->rq[i].vq, mask); + virtqueue_set_affinity(vi->sq[i].vq, mask); + __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, false); + cpumask_clear(mask); } vi->affinity_hint_set = true; + free_cpumask_var(mask); } static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) diff --git a/drivers/net/wimax/i2400m/usb-fw.c b/drivers/net/wimax/i2400m/usb-fw.c index 502c346aa790..529ebca1e9e1 100644 --- a/drivers/net/wimax/i2400m/usb-fw.c +++ b/drivers/net/wimax/i2400m/usb-fw.c @@ -130,12 +130,12 @@ retry: dev_err(dev, "BM-CMD: too many stalls in " "URB; resetting device\n"); usb_queue_reset_device(i2400mu->usb_iface); - /* fallthrough */ } else { usb_clear_halt(i2400mu->usb_dev, pipe); msleep(10); /* give the device some time */ goto retry; } + /* fall through */ case -EINVAL: /* while removing driver */ case -ENODEV: /* dev disconnect ... */ case -ENOENT: /* just ignore it */ diff --git a/drivers/net/wimax/i2400m/usb-tx.c b/drivers/net/wimax/i2400m/usb-tx.c index 99ef81b3d5a5..3a0e7226768a 100644 --- a/drivers/net/wimax/i2400m/usb-tx.c +++ b/drivers/net/wimax/i2400m/usb-tx.c @@ -131,12 +131,12 @@ retry: dev_err(dev, "BM-CMD: too many stalls in " "URB; resetting device\n"); usb_queue_reset_device(i2400mu->usb_iface); - /* fallthrough */ } else { usb_clear_halt(i2400mu->usb_dev, usb_pipe); msleep(10); /* give the device some time */ goto retry; } + /* fall through */ case -EINVAL: /* while removing driver */ case -ENODEV: /* dev disconnect ... */ case -ENOENT: /* just ignore it */ diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c index fa39ffffd34d..c9bd0e2b5db7 100644 --- a/drivers/net/wireless/ath/ath10k/ahb.c +++ b/drivers/net/wireless/ath/ath10k/ahb.c @@ -133,11 +133,8 @@ static void ath10k_ahb_clock_deinit(struct ath10k *ar) static int ath10k_ahb_clock_enable(struct ath10k *ar) { struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar); - struct device *dev; int ret; - dev = &ar_ahb->pdev->dev; - if (IS_ERR_OR_NULL(ar_ahb->cmd_clk) || IS_ERR_OR_NULL(ar_ahb->ref_clk) || IS_ERR_OR_NULL(ar_ahb->rtc_clk)) { @@ -451,12 +448,10 @@ static int ath10k_ahb_resource_init(struct ath10k *ar) { struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar); struct platform_device *pdev; - struct device *dev; struct resource *res; int ret; pdev = ar_ahb->pdev; - dev = &pdev->dev; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) { diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c index 85c58ebbfb26..c40cd129afe7 100644 --- a/drivers/net/wireless/ath/ath10k/core.c +++ b/drivers/net/wireless/ath/ath10k/core.c @@ -2095,6 +2095,7 @@ static int ath10k_core_init_firmware_features(struct ath10k *ar) ar->fw_stats_req_mask = WMI_STAT_PDEV | WMI_STAT_VDEV | WMI_STAT_PEER; ar->max_spatial_stream = WMI_MAX_SPATIAL_STREAM; + ar->wmi.mgmt_max_num_pending_tx = TARGET_TLV_MGMT_NUM_MSDU_DESC; break; case ATH10K_FW_WMI_OP_VERSION_10_4: ar->max_num_peers = TARGET_10_4_NUM_PEERS; diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index 427ee5752bb0..9feea02e7d37 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -186,6 +186,11 @@ struct ath10k_wmi { const struct wmi_ops *ops; const struct wmi_peer_flags_map *peer_flags; + u32 mgmt_max_num_pending_tx; + + /* Protected by data_lock */ + struct idr mgmt_pending_tx; + u32 num_mem_chunks; u32 rx_decap_mode; struct ath10k_mem_chunk mem_chunks[WMI_MAX_MEM_REQS]; diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c index 4926722e0c0d..0baaad90b8d1 100644 --- a/drivers/net/wireless/ath/ath10k/debug.c +++ b/drivers/net/wireless/ath/ath10k/debug.c @@ -2297,6 +2297,52 @@ static const struct file_operations fops_tpc_stats_final = { .llseek = default_llseek, }; +static ssize_t ath10k_write_warm_hw_reset(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath10k *ar = file->private_data; + int ret; + bool val; + + if (kstrtobool_from_user(user_buf, count, &val)) + return -EFAULT; + + if (!val) + return -EINVAL; + + mutex_lock(&ar->conf_mutex); + + if (ar->state != ATH10K_STATE_ON) { + ret = -ENETDOWN; + goto exit; + } + + if (!(test_bit(WMI_SERVICE_RESET_CHIP, ar->wmi.svc_map))) + ath10k_warn(ar, "wmi service for reset chip is not available\n"); + + ret = ath10k_wmi_pdev_set_param(ar, ar->wmi.pdev_param->pdev_reset, + WMI_RST_MODE_WARM_RESET); + + if (ret) { + ath10k_warn(ar, "failed to enable warm hw reset: %d\n", ret); + goto exit; + } + + ret = count; + +exit: + mutex_unlock(&ar->conf_mutex); + return ret; +} + +static const struct file_operations fops_warm_hw_reset = { + .write = ath10k_write_warm_hw_reset, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + int ath10k_debug_create(struct ath10k *ar) { ar->debug.cal_data = vzalloc(ATH10K_DEBUG_CAL_DATA_LEN); @@ -2425,6 +2471,9 @@ int ath10k_debug_register(struct ath10k *ar) ar->debug.debugfs_phy, ar, &fops_tpc_stats_final); + debugfs_create_file("warm_hw_reset", 0600, ar->debug.debugfs_phy, ar, + &fops_warm_hw_reset); + return 0; } diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c index be5b52aaffa6..7cff0d52338f 100644 --- a/drivers/net/wireless/ath/ath10k/htt_tx.c +++ b/drivers/net/wireless/ath/ath10k/htt_tx.c @@ -208,10 +208,10 @@ int ath10k_htt_tx_alloc_msdu_id(struct ath10k_htt *htt, struct sk_buff *skb) struct ath10k *ar = htt->ar; int ret; - lockdep_assert_held(&htt->tx_lock); - + spin_lock_bh(&htt->tx_lock); ret = idr_alloc(&htt->pending_tx, skb, 0, htt->max_num_pending_tx, GFP_ATOMIC); + spin_unlock_bh(&htt->tx_lock); ath10k_dbg(ar, ATH10K_DBG_HTT, "htt tx alloc msdu_id %d\n", ret); @@ -1077,9 +1077,7 @@ int ath10k_htt_mgmt_tx(struct ath10k_htt *htt, struct sk_buff *msdu) len += sizeof(cmd->hdr); len += sizeof(cmd->mgmt_tx); - spin_lock_bh(&htt->tx_lock); res = ath10k_htt_tx_alloc_msdu_id(htt, msdu); - spin_unlock_bh(&htt->tx_lock); if (res < 0) goto err; @@ -1161,9 +1159,7 @@ static int ath10k_htt_tx_32(struct ath10k_htt *htt, struct htt_msdu_ext_desc *ext_desc = NULL; struct htt_msdu_ext_desc *ext_desc_t = NULL; - spin_lock_bh(&htt->tx_lock); res = ath10k_htt_tx_alloc_msdu_id(htt, msdu); - spin_unlock_bh(&htt->tx_lock); if (res < 0) goto err; @@ -1363,9 +1359,7 @@ static int ath10k_htt_tx_64(struct ath10k_htt *htt, struct htt_msdu_ext_desc_64 *ext_desc = NULL; struct htt_msdu_ext_desc_64 *ext_desc_t = NULL; - spin_lock_bh(&htt->tx_lock); res = ath10k_htt_tx_alloc_msdu_id(htt, msdu); - spin_unlock_bh(&htt->tx_lock); if (res < 0) goto err; diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h index a274bd809a08..977f79ebb4fd 100644 --- a/drivers/net/wireless/ath/ath10k/hw.h +++ b/drivers/net/wireless/ath/ath10k/hw.h @@ -699,6 +699,7 @@ ath10k_rx_desc_get_l3_pad_bytes(struct ath10k_hw_params *hw, #define TARGET_TLV_NUM_TIDS ((TARGET_TLV_NUM_PEERS) * 2) #define TARGET_TLV_NUM_MSDU_DESC (1024 + 32) #define TARGET_TLV_NUM_WOW_PATTERNS 22 +#define TARGET_TLV_MGMT_NUM_MSDU_DESC (50) /* Target specific defines for WMI-HL-1.0 firmware */ #define TARGET_HL_10_TLV_NUM_PEERS 14 diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c index 95243b48a179..90f9372dec25 100644 --- a/drivers/net/wireless/ath/ath10k/mac.c +++ b/drivers/net/wireless/ath/ath10k/mac.c @@ -101,6 +101,8 @@ static struct ieee80211_rate ath10k_rates_rev2[] = { #define ath10k_g_rates_rev2 (ath10k_rates_rev2 + 0) #define ath10k_g_rates_rev2_size (ARRAY_SIZE(ath10k_rates_rev2)) +#define ath10k_wmi_legacy_rates ath10k_rates + static bool ath10k_mac_bitrate_is_cck(int bitrate) { switch (bitrate) { @@ -3085,6 +3087,13 @@ static int ath10k_update_channel_list(struct ath10k *ar) passive = channel->flags & IEEE80211_CHAN_NO_IR; ch->passive = passive; + /* the firmware is ignoring the "radar" flag of the + * channel and is scanning actively using Probe Requests + * on "Radar detection"/DFS channels which are not + * marked as "available" + */ + ch->passive |= ch->chan_radar; + ch->freq = channel->center_freq; ch->band_center_freq1 = channel->center_freq; ch->min_power = 0; @@ -5439,8 +5448,12 @@ static void ath10k_bss_info_changed(struct ieee80211_hw *hw, { struct ath10k *ar = hw->priv; struct ath10k_vif *arvif = (void *)vif->drv_priv; - int ret = 0; + struct cfg80211_chan_def def; u32 vdev_param, pdev_param, slottime, preamble; + u16 bitrate, hw_value; + u8 rate; + int rateidx, ret = 0; + enum nl80211_band band; mutex_lock(&ar->conf_mutex); @@ -5608,6 +5621,44 @@ static void ath10k_bss_info_changed(struct ieee80211_hw *hw, arvif->vdev_id, ret); } + if (changed & BSS_CHANGED_MCAST_RATE && + !WARN_ON(ath10k_mac_vif_chan(arvif->vif, &def))) { + band = def.chan->band; + rateidx = vif->bss_conf.mcast_rate[band] - 1; + + if (ar->phy_capability & WHAL_WLAN_11A_CAPABILITY) + rateidx += ATH10K_MAC_FIRST_OFDM_RATE_IDX; + + bitrate = ath10k_wmi_legacy_rates[rateidx].bitrate; + hw_value = ath10k_wmi_legacy_rates[rateidx].hw_value; + if (ath10k_mac_bitrate_is_cck(bitrate)) + preamble = WMI_RATE_PREAMBLE_CCK; + else + preamble = WMI_RATE_PREAMBLE_OFDM; + + rate = ATH10K_HW_RATECODE(hw_value, 0, preamble); + + ath10k_dbg(ar, ATH10K_DBG_MAC, + "mac vdev %d mcast_rate %x\n", + arvif->vdev_id, rate); + + vdev_param = ar->wmi.vdev_param->mcast_data_rate; + ret = ath10k_wmi_vdev_set_param(ar, arvif->vdev_id, + vdev_param, rate); + if (ret) + ath10k_warn(ar, + "failed to set mcast rate on vdev %i: %d\n", + arvif->vdev_id, ret); + + vdev_param = ar->wmi.vdev_param->bcast_data_rate; + ret = ath10k_wmi_vdev_set_param(ar, arvif->vdev_id, + vdev_param, rate); + if (ret) + ath10k_warn(ar, + "failed to set bcast rate on vdev %i: %d\n", + arvif->vdev_id, ret); + } + mutex_unlock(&ar->conf_mutex); } @@ -6063,13 +6114,13 @@ static void ath10k_sta_rc_update_wk(struct work_struct *wk) mode = chan_to_phymode(&def); ath10k_dbg(ar, ATH10K_DBG_MAC, "mac update sta %pM peer bw %d phymode %d\n", - sta->addr, bw, mode); + sta->addr, bw, mode); err = ath10k_wmi_peer_set_param(ar, arvif->vdev_id, sta->addr, - WMI_PEER_PHYMODE, mode); + WMI_PEER_PHYMODE, mode); if (err) { ath10k_warn(ar, "failed to update STA %pM peer phymode %d: %d\n", - sta->addr, mode, err); + sta->addr, mode, err); goto exit; } @@ -6935,7 +6986,6 @@ ath10k_mac_bitrate_mask_get_single_rate(struct ath10k *ar, const struct cfg80211_bitrate_mask *mask, u8 *rate, u8 *nss) { - struct ieee80211_supported_band *sband = &ar->mac.sbands[band]; int rate_idx; int i; u16 bitrate; @@ -6945,8 +6995,11 @@ ath10k_mac_bitrate_mask_get_single_rate(struct ath10k *ar, if (hweight32(mask->control[band].legacy) == 1) { rate_idx = ffs(mask->control[band].legacy) - 1; - hw_rate = sband->bitrates[rate_idx].hw_value; - bitrate = sband->bitrates[rate_idx].bitrate; + if (ar->phy_capability & WHAL_WLAN_11A_CAPABILITY) + rate_idx += ATH10K_MAC_FIRST_OFDM_RATE_IDX; + + hw_rate = ath10k_wmi_legacy_rates[rate_idx].hw_value; + bitrate = ath10k_wmi_legacy_rates[rate_idx].bitrate; if (ath10k_mac_bitrate_is_cck(bitrate)) preamble = WMI_RATE_PREAMBLE_CCK; diff --git a/drivers/net/wireless/ath/ath10k/wmi-ops.h b/drivers/net/wireless/ath/ath10k/wmi-ops.h index 5ecce04005d2..7fd63bbf8e24 100644 --- a/drivers/net/wireless/ath/ath10k/wmi-ops.h +++ b/drivers/net/wireless/ath/ath10k/wmi-ops.h @@ -31,6 +31,8 @@ struct wmi_ops { struct wmi_scan_ev_arg *arg); int (*pull_mgmt_rx)(struct ath10k *ar, struct sk_buff *skb, struct wmi_mgmt_rx_ev_arg *arg); + int (*pull_mgmt_tx_compl)(struct ath10k *ar, struct sk_buff *skb, + struct wmi_tlv_mgmt_tx_compl_ev_arg *arg); int (*pull_ch_info)(struct ath10k *ar, struct sk_buff *skb, struct wmi_ch_info_ev_arg *arg); int (*pull_vdev_start)(struct ath10k *ar, struct sk_buff *skb, @@ -262,6 +264,16 @@ ath10k_wmi_pull_scan(struct ath10k *ar, struct sk_buff *skb, } static inline int +ath10k_wmi_pull_mgmt_tx_compl(struct ath10k *ar, struct sk_buff *skb, + struct wmi_tlv_mgmt_tx_compl_ev_arg *arg) +{ + if (!ar->wmi.ops->pull_mgmt_tx_compl) + return -EOPNOTSUPP; + + return ar->wmi.ops->pull_mgmt_tx_compl(ar, skb, arg); +} + +static inline int ath10k_wmi_pull_mgmt_rx(struct ath10k *ar, struct sk_buff *skb, struct wmi_mgmt_rx_ev_arg *arg) { diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.c b/drivers/net/wireless/ath/ath10k/wmi-tlv.c index b04f86f8038a..cdc1e64d52ad 100644 --- a/drivers/net/wireless/ath/ath10k/wmi-tlv.c +++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.c @@ -618,6 +618,9 @@ static void ath10k_wmi_tlv_op_rx(struct ath10k *ar, struct sk_buff *skb) case WMI_TLV_TDLS_PEER_EVENTID: ath10k_wmi_event_tdls_peer(ar, skb); break; + case WMI_TLV_MGMT_TX_COMPLETION_EVENTID: + ath10k_wmi_event_mgmt_tx_compl(ar, skb); + break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; @@ -659,6 +662,31 @@ static int ath10k_wmi_tlv_op_pull_scan_ev(struct ath10k *ar, return 0; } +static int +ath10k_wmi_tlv_op_pull_mgmt_tx_compl_ev(struct ath10k *ar, struct sk_buff *skb, + struct wmi_tlv_mgmt_tx_compl_ev_arg *arg) +{ + const void **tb; + const struct wmi_tlv_mgmt_tx_compl_ev *ev; + int ret; + + tb = ath10k_wmi_tlv_parse_alloc(ar, skb->data, skb->len, GFP_ATOMIC); + if (IS_ERR(tb)) { + ret = PTR_ERR(tb); + ath10k_warn(ar, "failed to parse tlv: %d\n", ret); + return ret; + } + + ev = tb[WMI_TLV_TAG_STRUCT_MGMT_TX_COMPL_EVENT]; + + arg->desc_id = ev->desc_id; + arg->status = ev->status; + arg->pdev_id = ev->pdev_id; + + kfree(tb); + return 0; +} + static int ath10k_wmi_tlv_op_pull_mgmt_rx_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_mgmt_rx_ev_arg *arg) @@ -1586,6 +1614,11 @@ static struct sk_buff *ath10k_wmi_tlv_op_gen_init(struct ath10k *ar) cfg->keep_alive_pattern_size = __cpu_to_le32(0); cfg->max_tdls_concurrent_sleep_sta = __cpu_to_le32(1); cfg->max_tdls_concurrent_buffer_sta = __cpu_to_le32(1); + cfg->wmi_send_separate = __cpu_to_le32(0); + cfg->num_ocb_vdevs = __cpu_to_le32(0); + cfg->num_ocb_channels = __cpu_to_le32(0); + cfg->num_ocb_schedules = __cpu_to_le32(0); + cfg->host_capab = __cpu_to_le32(0); ath10k_wmi_put_host_mem_chunks(ar, chunks); @@ -1787,7 +1820,6 @@ ath10k_wmi_tlv_op_gen_vdev_start(struct ath10k *ar, { struct wmi_tlv_vdev_start_cmd *cmd; struct wmi_channel *ch; - struct wmi_p2p_noa_descriptor *noa; struct wmi_tlv *tlv; struct sk_buff *skb; size_t len; @@ -1845,7 +1877,6 @@ ath10k_wmi_tlv_op_gen_vdev_start(struct ath10k *ar, tlv = ptr; tlv->tag = __cpu_to_le16(WMI_TLV_TAG_ARRAY_STRUCT); tlv->len = 0; - noa = (void *)tlv->value; /* Note: This is a nested TLV containing: * [wmi_tlv][wmi_p2p_noa_descriptor][wmi_tlv].. @@ -2607,6 +2638,30 @@ ath10k_wmi_tlv_op_gen_request_stats(struct ath10k *ar, u32 stats_mask) return skb; } +static int +ath10k_wmi_mgmt_tx_alloc_msdu_id(struct ath10k *ar, struct sk_buff *skb, + dma_addr_t paddr) +{ + struct ath10k_wmi *wmi = &ar->wmi; + struct ath10k_mgmt_tx_pkt_addr *pkt_addr; + int ret; + + pkt_addr = kmalloc(sizeof(*pkt_addr), GFP_ATOMIC); + if (!pkt_addr) + return -ENOMEM; + + pkt_addr->vaddr = skb; + pkt_addr->paddr = paddr; + + spin_lock_bh(&ar->data_lock); + ret = idr_alloc(&wmi->mgmt_pending_tx, pkt_addr, 0, + wmi->mgmt_max_num_pending_tx, GFP_ATOMIC); + spin_unlock_bh(&ar->data_lock); + + ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt tx alloc msdu_id ret %d\n", ret); + return ret; +} + static struct sk_buff * ath10k_wmi_tlv_op_gen_mgmt_tx_send(struct ath10k *ar, struct sk_buff *msdu, dma_addr_t paddr) @@ -2618,9 +2673,9 @@ ath10k_wmi_tlv_op_gen_mgmt_tx_send(struct ath10k *ar, struct sk_buff *msdu, u32 buf_len = msdu->len; struct wmi_tlv *tlv; struct sk_buff *skb; + int len, desc_id; u32 vdev_id; void *ptr; - int len; if (!cb->vif) return ERR_PTR(-EINVAL); @@ -2651,13 +2706,17 @@ ath10k_wmi_tlv_op_gen_mgmt_tx_send(struct ath10k *ar, struct sk_buff *msdu, if (!skb) return ERR_PTR(-ENOMEM); + desc_id = ath10k_wmi_mgmt_tx_alloc_msdu_id(ar, msdu, paddr); + if (desc_id < 0) + goto err_free_skb; + ptr = (void *)skb->data; tlv = ptr; tlv->tag = __cpu_to_le16(WMI_TLV_TAG_STRUCT_MGMT_TX_CMD); tlv->len = __cpu_to_le16(sizeof(*cmd)); cmd = (void *)tlv->value; cmd->vdev_id = __cpu_to_le32(vdev_id); - cmd->desc_id = 0; + cmd->desc_id = __cpu_to_le32(desc_id); cmd->chanfreq = 0; cmd->buf_len = __cpu_to_le32(buf_len); cmd->frame_len = __cpu_to_le32(msdu->len); @@ -2674,6 +2733,10 @@ ath10k_wmi_tlv_op_gen_mgmt_tx_send(struct ath10k *ar, struct sk_buff *msdu, memcpy(ptr, msdu->data, buf_len); return skb; + +err_free_skb: + dev_kfree_skb(skb); + return ERR_PTR(desc_id); } static struct sk_buff * @@ -2702,7 +2765,8 @@ ath10k_wmi_tlv_op_gen_force_fw_hang(struct ath10k *ar, static struct sk_buff * ath10k_wmi_tlv_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable, - u32 log_level) { + u32 log_level) +{ struct wmi_tlv_dbglog_cmd *cmd; struct wmi_tlv *tlv; struct sk_buff *skb; @@ -3837,6 +3901,7 @@ static const struct wmi_ops wmi_tlv_ops = { .pull_scan = ath10k_wmi_tlv_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_tlv_op_pull_mgmt_rx_ev, + .pull_mgmt_tx_compl = ath10k_wmi_tlv_op_pull_mgmt_tx_compl_ev, .pull_ch_info = ath10k_wmi_tlv_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_tlv_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_tlv_op_pull_peer_kick_ev, diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.h b/drivers/net/wireless/ath/ath10k/wmi-tlv.h index 3e1e340cd834..4f0c20c90642 100644 --- a/drivers/net/wireless/ath/ath10k/wmi-tlv.h +++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.h @@ -320,6 +320,7 @@ enum wmi_tlv_event_id { WMI_TLV_TBTTOFFSET_UPDATE_EVENTID, WMI_TLV_OFFLOAD_BCN_TX_STATUS_EVENTID, WMI_TLV_OFFLOAD_PROB_RESP_TX_STATUS_EVENTID, + WMI_TLV_MGMT_TX_COMPLETION_EVENTID, WMI_TLV_TX_DELBA_COMPLETE_EVENTID = WMI_TLV_EV(WMI_TLV_GRP_BA_NEG), WMI_TLV_TX_ADDBA_COMPLETE_EVENTID, WMI_TLV_BA_RSP_SSN_EVENTID, @@ -1573,6 +1574,17 @@ struct wmi_tlv { u8 value[0]; } __packed; +struct ath10k_mgmt_tx_pkt_addr { + void *vaddr; + dma_addr_t paddr; +}; + +struct wmi_tlv_mgmt_tx_compl_ev { + __le32 desc_id; + __le32 status; + __le32 pdev_id; +}; + #define WMI_TLV_MGMT_RX_NUM_RSSI 4 struct wmi_tlv_mgmt_rx_ev { @@ -1670,6 +1682,11 @@ struct wmi_tlv_resource_config { __le32 keep_alive_pattern_size; __le32 max_tdls_concurrent_sleep_sta; __le32 max_tdls_concurrent_buffer_sta; + __le32 wmi_send_separate; + __le32 num_ocb_vdevs; + __le32 num_ocb_channels; + __le32 num_ocb_schedules; + __le32 host_capab; } __packed; struct wmi_tlv_init_cmd { diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c index 877249ac6fd4..fd612d2905b0 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.c +++ b/drivers/net/wireless/ath/ath10k/wmi.c @@ -1333,7 +1333,7 @@ static struct wmi_pdev_param_map wmi_10_2_4_pdev_param_map = { .enable_per_tid_ampdu = WMI_PDEV_PARAM_UNSUPPORTED, .cca_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_PDEV_PARAM_UNSUPPORTED, - .pdev_reset = WMI_PDEV_PARAM_UNSUPPORTED, + .pdev_reset = WMI_10X_PDEV_PARAM_PDEV_RESET, .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED, .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED, .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED, @@ -2313,6 +2313,59 @@ static bool ath10k_wmi_rx_is_decrypted(struct ath10k *ar, return true; } +static int wmi_process_mgmt_tx_comp(struct ath10k *ar, u32 desc_id, + u32 status) +{ + struct ath10k_mgmt_tx_pkt_addr *pkt_addr; + struct ath10k_wmi *wmi = &ar->wmi; + struct ieee80211_tx_info *info; + struct sk_buff *msdu; + int ret; + + spin_lock_bh(&ar->data_lock); + + pkt_addr = idr_find(&wmi->mgmt_pending_tx, desc_id); + if (!pkt_addr) { + ath10k_warn(ar, "received mgmt tx completion for invalid msdu_id: %d\n", + desc_id); + ret = -ENOENT; + goto out; + } + + msdu = pkt_addr->vaddr; + dma_unmap_single(ar->dev, pkt_addr->paddr, + msdu->len, DMA_FROM_DEVICE); + info = IEEE80211_SKB_CB(msdu); + info->flags |= status; + ieee80211_tx_status_irqsafe(ar->hw, msdu); + + ret = 0; + +out: + idr_remove(&wmi->mgmt_pending_tx, desc_id); + spin_unlock_bh(&ar->data_lock); + return ret; +} + +int ath10k_wmi_event_mgmt_tx_compl(struct ath10k *ar, struct sk_buff *skb) +{ + struct wmi_tlv_mgmt_tx_compl_ev_arg arg; + int ret; + + ret = ath10k_wmi_pull_mgmt_tx_compl(ar, skb, &arg); + if (ret) { + ath10k_warn(ar, "failed to parse mgmt comp event: %d\n", ret); + return ret; + } + + wmi_process_mgmt_tx_comp(ar, __le32_to_cpu(arg.desc_id), + __le32_to_cpu(arg.status)); + + ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv evnt mgmt tx completion\n"); + + return 0; +} + int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_mgmt_rx_ev_arg arg = {}; @@ -9073,6 +9126,11 @@ int ath10k_wmi_attach(struct ath10k *ar) INIT_WORK(&ar->radar_confirmation_work, ath10k_radar_confirmation_work); + if (test_bit(ATH10K_FW_FEATURE_MGMT_TX_BY_REF, + ar->running_fw->fw_file.fw_features)) { + idr_init(&ar->wmi.mgmt_pending_tx); + } + return 0; } @@ -9091,8 +9149,35 @@ void ath10k_wmi_free_host_mem(struct ath10k *ar) ar->wmi.num_mem_chunks = 0; } +static int ath10k_wmi_mgmt_tx_clean_up_pending(int msdu_id, void *ptr, + void *ctx) +{ + struct ath10k_mgmt_tx_pkt_addr *pkt_addr = ptr; + struct ath10k *ar = ctx; + struct sk_buff *msdu; + + ath10k_dbg(ar, ATH10K_DBG_WMI, + "force cleanup mgmt msdu_id %hu\n", msdu_id); + + msdu = pkt_addr->vaddr; + dma_unmap_single(ar->dev, pkt_addr->paddr, + msdu->len, DMA_FROM_DEVICE); + ieee80211_free_txskb(ar->hw, msdu); + + return 0; +} + void ath10k_wmi_detach(struct ath10k *ar) { + if (test_bit(ATH10K_FW_FEATURE_MGMT_TX_BY_REF, + ar->running_fw->fw_file.fw_features)) { + spin_lock_bh(&ar->data_lock); + idr_for_each(&ar->wmi.mgmt_pending_tx, + ath10k_wmi_mgmt_tx_clean_up_pending, ar); + idr_destroy(&ar->wmi.mgmt_pending_tx); + spin_unlock_bh(&ar->data_lock); + } + cancel_work_sync(&ar->svc_rdy_work); if (ar->svc_rdy_skb) diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h index d68afb65402a..36220258e3c7 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.h +++ b/drivers/net/wireless/ath/ath10k/wmi.h @@ -462,6 +462,7 @@ static inline char *wmi_service_name(int service_id) SVCSTR(WMI_SERVICE_HTT_MGMT_TX_COMP_VALID_FLAGS); SVCSTR(WMI_SERVICE_HOST_DFS_CHECK_SUPPORT); SVCSTR(WMI_SERVICE_TPC_STATS_FINAL); + SVCSTR(WMI_SERVICE_RESET_CHIP); default: return NULL; } @@ -3934,7 +3935,11 @@ enum wmi_10x_pdev_param { WMI_10X_PDEV_PARAM_REMOVE_MCAST2UCAST_BUFFER, WMI_10X_PDEV_PARAM_PEER_STA_PS_STATECHG_ENABLE, WMI_10X_PDEV_PARAM_RTS_FIXED_RATE, - WMI_10X_PDEV_PARAM_CAL_PERIOD + WMI_10X_PDEV_PARAM_CAL_PERIOD, + WMI_10X_PDEV_PARAM_ATF_STRICT_SCH, + WMI_10X_PDEV_PARAM_ATF_SCHED_DURATION, + WMI_10X_PDEV_PARAM_SET_PROMISC_MODE_CMDID, + WMI_10X_PDEV_PARAM_PDEV_RESET }; enum wmi_10_4_pdev_param { @@ -6501,6 +6506,15 @@ struct wmi_force_fw_hang_cmd { __le32 delay_ms; } __packed; +enum wmi_pdev_reset_mode_type { + WMI_RST_MODE_TX_FLUSH = 1, + WMI_RST_MODE_WARM_RESET, + WMI_RST_MODE_COLD_RESET, + WMI_RST_MODE_WARM_RESET_RESTORE_CAL, + WMI_RST_MODE_COLD_RESET_RESTORE_CAL, + WMI_RST_MODE_MAX, +}; + enum ath10k_dbglog_level { ATH10K_DBGLOG_LEVEL_VERBOSE = 0, ATH10K_DBGLOG_LEVEL_INFO = 1, @@ -6600,6 +6614,12 @@ struct wmi_scan_ev_arg { __le32 vdev_id; }; +struct wmi_tlv_mgmt_tx_compl_ev_arg { + __le32 desc_id; + __le32 status; + __le32 pdev_id; +}; + struct wmi_mgmt_rx_ev_arg { __le32 channel; __le32 snr; @@ -7071,6 +7091,7 @@ int ath10k_wmi_start_scan_verify(const struct wmi_start_scan_arg *arg); int ath10k_wmi_event_scan(struct ath10k *ar, struct sk_buff *skb); int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb); +int ath10k_wmi_event_mgmt_tx_compl(struct ath10k *ar, struct sk_buff *skb); void ath10k_wmi_event_chan_info(struct ath10k *ar, struct sk_buff *skb); void ath10k_wmi_event_echo(struct ath10k *ar, struct sk_buff *skb); int ath10k_wmi_event_debug_mesg(struct ath10k *ar, struct sk_buff *skb); diff --git a/drivers/net/wireless/ath/ath5k/phy.c b/drivers/net/wireless/ath/ath5k/phy.c index b1b8bc326830..ae08572c4b58 100644 --- a/drivers/net/wireless/ath/ath5k/phy.c +++ b/drivers/net/wireless/ath/ath5k/phy.c @@ -483,7 +483,6 @@ static u32 ath5k_hw_rf_gainf_corr(struct ath5k_hw *ah) { u32 mix, step; - u32 *rf; const struct ath5k_gain_opt *go; const struct ath5k_gain_opt_step *g_step; const struct ath5k_rf_reg *rf_regs; @@ -502,7 +501,6 @@ ath5k_hw_rf_gainf_corr(struct ath5k_hw *ah) if (ah->ah_rf_banks == NULL) return 0; - rf = ah->ah_rf_banks; ah->ah_gain.g_f_corr = 0; /* No VGA (Variable Gain Amplifier) override, skip */ @@ -549,13 +547,10 @@ ath5k_hw_rf_check_gainf_readback(struct ath5k_hw *ah) { const struct ath5k_rf_reg *rf_regs; u32 step, mix_ovr, level[4]; - u32 *rf; if (ah->ah_rf_banks == NULL) return false; - rf = ah->ah_rf_banks; - if (ah->ah_radio == AR5K_RF5111) { rf_regs = rf_regs_5111; diff --git a/drivers/net/wireless/ath/ath6kl/bmi.c b/drivers/net/wireless/ath/ath6kl/bmi.c index 334dbd834b3a..bde5a10d470c 100644 --- a/drivers/net/wireless/ath/ath6kl/bmi.c +++ b/drivers/net/wireless/ath/ath6kl/bmi.c @@ -534,7 +534,7 @@ int ath6kl_bmi_init(struct ath6kl *ar) /* cmd + addr + len + data_size */ ar->bmi.max_cmd_size = ar->bmi.max_data_size + (sizeof(u32) * 3); - ar->bmi.cmd_buf = kzalloc(ar->bmi.max_cmd_size, GFP_ATOMIC); + ar->bmi.cmd_buf = kzalloc(ar->bmi.max_cmd_size, GFP_KERNEL); if (!ar->bmi.cmd_buf) return -ENOMEM; diff --git a/drivers/net/wireless/ath/ath6kl/htc_pipe.c b/drivers/net/wireless/ath/ath6kl/htc_pipe.c index 546243e11737..434b66829646 100644 --- a/drivers/net/wireless/ath/ath6kl/htc_pipe.c +++ b/drivers/net/wireless/ath/ath6kl/htc_pipe.c @@ -746,10 +746,8 @@ static int ath6kl_htc_pipe_tx_complete(struct ath6kl *ar, struct sk_buff *skb) struct htc_endpoint *ep; struct htc_packet *packet; u8 ep_id, *netdata; - u32 netlen; netdata = skb->data; - netlen = skb->len; htc_hdr = (struct htc_frame_hdr *) netdata; @@ -855,12 +853,8 @@ static int htc_process_trailer(struct htc_target *target, u8 *buffer, { struct htc_credit_report *report; struct htc_record_hdr *record; - u8 *record_buf, *orig_buf; - int orig_len, status; - - orig_buf = buffer; - orig_len = len; - status = 0; + u8 *record_buf; + int status = 0; while (len > 0) { if (len < sizeof(struct htc_record_hdr)) { diff --git a/drivers/net/wireless/ath/ath6kl/main.c b/drivers/net/wireless/ath/ath6kl/main.c index 808fb30be9ad..0c61dbaa62a4 100644 --- a/drivers/net/wireless/ath/ath6kl/main.c +++ b/drivers/net/wireless/ath/ath6kl/main.c @@ -272,7 +272,7 @@ int ath6kl_read_fwlogs(struct ath6kl *ar) { struct ath6kl_dbglog_hdr debug_hdr; struct ath6kl_dbglog_buf debug_buf; - u32 address, length, dropped, firstbuf, debug_hdr_addr; + u32 address, length, firstbuf, debug_hdr_addr; int ret, loop; u8 *buf; @@ -303,7 +303,6 @@ int ath6kl_read_fwlogs(struct ath6kl *ar) address = TARG_VTOP(ar->target_type, le32_to_cpu(debug_hdr.dbuf_addr)); firstbuf = address; - dropped = le32_to_cpu(debug_hdr.dropped); ret = ath6kl_diag_read(ar, address, &debug_buf, sizeof(debug_buf)); if (ret) goto out; diff --git a/drivers/net/wireless/ath/ath6kl/txrx.c b/drivers/net/wireless/ath/ath6kl/txrx.c index 618d12ed4b40..b22ed499f7ba 100644 --- a/drivers/net/wireless/ath/ath6kl/txrx.c +++ b/drivers/net/wireless/ath/ath6kl/txrx.c @@ -1701,7 +1701,6 @@ void aggr_recv_addba_req_evt(struct ath6kl_vif *vif, u8 tid_mux, u16 seq_no, struct ath6kl_sta *sta; struct aggr_info_conn *aggr_conn = NULL; struct rxtid *rxtid; - struct rxtid_stats *stats; u16 hold_q_size; u8 tid, aid; @@ -1722,7 +1721,6 @@ void aggr_recv_addba_req_evt(struct ath6kl_vif *vif, u8 tid_mux, u16 seq_no, return; rxtid = &aggr_conn->rx_tid[tid]; - stats = &aggr_conn->stat[tid]; if (win_sz < AGGR_WIN_SZ_MIN || win_sz > AGGR_WIN_SZ_MAX) ath6kl_dbg(ATH6KL_DBG_WLAN_RX, "%s: win_sz %d, tid %d\n", diff --git a/drivers/net/wireless/ath/ath9k/ar9002_calib.c b/drivers/net/wireless/ath/ath9k/ar9002_calib.c index 50fcd343c41a..fd9db8ca99d7 100644 --- a/drivers/net/wireless/ath/ath9k/ar9002_calib.c +++ b/drivers/net/wireless/ath/ath9k/ar9002_calib.c @@ -676,10 +676,10 @@ static int ar9002_hw_calibrate(struct ath_hw *ah, struct ath9k_channel *chan, return 0; ah->cal_list_curr = currCal = currCal->calNext; - if (currCal->calState == CAL_WAITING) { + if (currCal->calState == CAL_WAITING) ath9k_hw_reset_calibration(ah, currCal); - return 0; - } + + return 0; } /* Do NF cal only at longer intervals */ diff --git a/drivers/net/wireless/ath/ath9k/ar9003_phy.c b/drivers/net/wireless/ath/ath9k/ar9003_phy.c index fe5102ca5010..98c5f524a360 100644 --- a/drivers/net/wireless/ath/ath9k/ar9003_phy.c +++ b/drivers/net/wireless/ath/ath9k/ar9003_phy.c @@ -1800,6 +1800,8 @@ static void ar9003_hw_spectral_scan_config(struct ath_hw *ah, static void ar9003_hw_spectral_scan_trigger(struct ath_hw *ah) { + REG_SET_BIT(ah, AR_PHY_SPECTRAL_SCAN, + AR_PHY_SPECTRAL_SCAN_ENABLE); /* Activate spectral scan */ REG_SET_BIT(ah, AR_PHY_SPECTRAL_SCAN, AR_PHY_SPECTRAL_SCAN_ACTIVE); diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c index 32fb85e076d6..bb319f22761f 100644 --- a/drivers/net/wireless/ath/ath9k/hw.c +++ b/drivers/net/wireless/ath/ath9k/hw.c @@ -2942,16 +2942,19 @@ void ath9k_hw_apply_txpower(struct ath_hw *ah, struct ath9k_channel *chan, struct ath_regulatory *reg = ath9k_hw_regulatory(ah); struct ieee80211_channel *channel; int chan_pwr, new_pwr; + u16 ctl = NO_CTL; if (!chan) return; + if (!test) + ctl = ath9k_regd_get_ctl(reg, chan); + channel = chan->chan; chan_pwr = min_t(int, channel->max_power * 2, MAX_RATE_POWER); new_pwr = min_t(int, chan_pwr, reg->power_limit); - ah->eep_ops->set_txpower(ah, chan, - ath9k_regd_get_ctl(reg, chan), + ah->eep_ops->set_txpower(ah, chan, ctl, get_antenna_gain(ah, chan), new_pwr, test); } diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c index 7fdb152be0bb..43b6c8508e49 100644 --- a/drivers/net/wireless/ath/ath9k/xmit.c +++ b/drivers/net/wireless/ath/ath9k/xmit.c @@ -62,7 +62,7 @@ static void ath_tx_rc_status(struct ath_softc *sc, struct ath_buf *bf, struct ath_tx_status *ts, int nframes, int nbad, int txok); static void ath_tx_update_baw(struct ath_softc *sc, struct ath_atx_tid *tid, - int seqno); + struct ath_buf *bf); static struct ath_buf *ath_tx_setup_buffer(struct ath_softc *sc, struct ath_txq *txq, struct ath_atx_tid *tid, @@ -86,7 +86,8 @@ static void ath_tx_status(struct ieee80211_hw *hw, struct sk_buff *skb) struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); struct ieee80211_sta *sta = info->status.status_driver_data[0]; - if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { + if (info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS | + IEEE80211_TX_STATUS_EOSP)) { ieee80211_tx_status(hw, skb); return; } @@ -295,7 +296,7 @@ static void ath_tx_flush_tid(struct ath_softc *sc, struct ath_atx_tid *tid) } if (fi->baw_tracked) { - ath_tx_update_baw(sc, tid, bf->bf_state.seqno); + ath_tx_update_baw(sc, tid, bf); sendbar = true; } @@ -311,10 +312,15 @@ static void ath_tx_flush_tid(struct ath_softc *sc, struct ath_atx_tid *tid) } static void ath_tx_update_baw(struct ath_softc *sc, struct ath_atx_tid *tid, - int seqno) + struct ath_buf *bf) { + struct ath_frame_info *fi = get_frame_info(bf->bf_mpdu); + u16 seqno = bf->bf_state.seqno; int index, cindex; + if (!fi->baw_tracked) + return; + index = ATH_BA_INDEX(tid->seq_start, seqno); cindex = (tid->baw_head + index) & (ATH_TID_MAX_BUFS - 1); @@ -335,6 +341,9 @@ static void ath_tx_addto_baw(struct ath_softc *sc, struct ath_atx_tid *tid, u16 seqno = bf->bf_state.seqno; int index, cindex; + if (fi->baw_tracked) + return; + index = ATH_BA_INDEX(tid->seq_start, seqno); cindex = (tid->baw_head + index) & (ATH_TID_MAX_BUFS - 1); __set_bit(cindex, tid->tx_buf); @@ -611,7 +620,7 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq, * complete the acked-ones/xretried ones; update * block-ack window */ - ath_tx_update_baw(sc, tid, seqno); + ath_tx_update_baw(sc, tid, bf); if (rc_update && (acked_cnt == 1 || txfail_cnt == 1)) { memcpy(tx_info->control.rates, rates, sizeof(rates)); @@ -641,7 +650,7 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq, * run out of tx buf. */ if (!tbf) { - ath_tx_update_baw(sc, tid, seqno); + ath_tx_update_baw(sc, tid, bf); ath_tx_complete_buf(sc, bf, txq, &bf_head, NULL, ts, @@ -969,7 +978,8 @@ ath_tx_get_tid_subframe(struct ath_softc *sc, struct ath_txq *txq, bf->bf_lastbf = bf; tx_info = IEEE80211_SKB_CB(skb); - tx_info->flags &= ~IEEE80211_TX_CTL_CLEAR_PS_FILT; + tx_info->flags &= ~(IEEE80211_TX_CTL_CLEAR_PS_FILT | + IEEE80211_TX_STATUS_EOSP); /* * No aggregation session is running, but there may be frames @@ -1009,11 +1019,14 @@ ath_tx_get_tid_subframe(struct ath_softc *sc, struct ath_txq *txq, INIT_LIST_HEAD(&bf_head); list_add(&bf->list, &bf_head); - ath_tx_update_baw(sc, tid, seqno); + ath_tx_update_baw(sc, tid, bf); ath_tx_complete_buf(sc, bf, txq, &bf_head, NULL, &ts, 0); continue; } + if (bf_isampdu(bf)) + ath_tx_addto_baw(sc, tid, bf); + return bf; } @@ -1071,8 +1084,6 @@ ath_tx_form_aggr(struct ath_softc *sc, struct ath_txq *txq, bf->bf_next = NULL; /* link buffers of this frame to the aggregate */ - if (!fi->baw_tracked) - ath_tx_addto_baw(sc, tid, bf); bf->bf_state.ndelim = ndelim; list_add_tail(&bf->list, bf_q); @@ -1659,6 +1670,22 @@ void ath_tx_aggr_wakeup(struct ath_softc *sc, struct ath_node *an) } } + +static void +ath9k_set_moredata(struct ath_softc *sc, struct ath_buf *bf, bool val) +{ + struct ieee80211_hdr *hdr; + u16 mask = cpu_to_le16(IEEE80211_FCTL_MOREDATA); + u16 mask_val = mask * val; + + hdr = (struct ieee80211_hdr *) bf->bf_mpdu->data; + if ((hdr->frame_control & mask) != mask_val) { + hdr->frame_control = (hdr->frame_control & ~mask) | mask_val; + dma_sync_single_for_device(sc->dev, bf->bf_buf_addr, + sizeof(*hdr), DMA_TO_DEVICE); + } +} + void ath9k_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta, u16 tids, int nframes, @@ -1689,12 +1716,11 @@ void ath9k_release_buffered_frames(struct ieee80211_hw *hw, if (!bf) break; + ath9k_set_moredata(sc, bf, true); list_add_tail(&bf->list, &bf_q); ath_set_rates(tid->an->vif, tid->an->sta, bf); - if (bf_isampdu(bf)) { - ath_tx_addto_baw(sc, tid, bf); + if (bf_isampdu(bf)) bf->bf_state.bf_type &= ~BUF_AGGR; - } if (bf_tail) bf_tail->bf_next = bf; @@ -1712,6 +1738,9 @@ void ath9k_release_buffered_frames(struct ieee80211_hw *hw, if (list_empty(&bf_q)) return; + if (!more_data) + ath9k_set_moredata(sc, bf_tail, false); + info = IEEE80211_SKB_CB(bf_tail->bf_mpdu); info->flags |= IEEE80211_TX_STATUS_EOSP; @@ -2407,7 +2436,6 @@ void ath_tx_cabq(struct ieee80211_hw *hw, struct ieee80211_vif *vif, .txq = sc->beacon.cabq }; struct ath_tx_info info = {}; - struct ieee80211_hdr *hdr; struct ath_buf *bf_tail = NULL; struct ath_buf *bf; LIST_HEAD(bf_q); @@ -2451,15 +2479,10 @@ void ath_tx_cabq(struct ieee80211_hw *hw, struct ieee80211_vif *vif, if (list_empty(&bf_q)) return; - bf = list_first_entry(&bf_q, struct ath_buf, list); - hdr = (struct ieee80211_hdr *) bf->bf_mpdu->data; - - if (hdr->frame_control & cpu_to_le16(IEEE80211_FCTL_MOREDATA)) { - hdr->frame_control &= ~cpu_to_le16(IEEE80211_FCTL_MOREDATA); - dma_sync_single_for_device(sc->dev, bf->bf_buf_addr, - sizeof(*hdr), DMA_TO_DEVICE); - } + bf = list_last_entry(&bf_q, struct ath_buf, list); + ath9k_set_moredata(sc, bf, false); + bf = list_first_entry(&bf_q, struct ath_buf, list); ath_txq_lock(sc, txctl.txq); ath_tx_fill_desc(sc, bf, txctl.txq, 0); ath_tx_txqaddbuf(sc, txctl.txq, &bf_q, false); diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c index e63b07830f2c..f79c337105cb 100644 --- a/drivers/net/wireless/ath/wil6210/cfg80211.c +++ b/drivers/net/wireless/ath/wil6210/cfg80211.c @@ -689,11 +689,12 @@ static int wil_cfg80211_scan(struct wiphy *wiphy, wil_dbg_misc(wil, "scan: wdev=0x%p iftype=%d\n", wdev, wdev->iftype); - /* check we are client side */ + /* scan is supported on client interfaces and on AP interface */ switch (wdev->iftype) { case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: case NL80211_IFTYPE_P2P_DEVICE: + case NL80211_IFTYPE_AP: break; default: return -EOPNOTSUPP; @@ -1089,18 +1090,51 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev, int rc; bool tx_status; - /* Note, currently we do not support the "wait" parameter, user-space - * must call remain_on_channel before mgmt_tx or listen on a channel - * another way (AP/PCP or connected station) - * in addition we need to check if specified "chan" argument is - * different from currently "listened" channel and fail if it is. + wil_dbg_misc(wil, "mgmt_tx: channel %d offchan %d, wait %d\n", + params->chan ? params->chan->hw_value : -1, + params->offchan, + params->wait); + + /* Note, currently we support the "wait" parameter only on AP mode. + * In other modes, user-space must call remain_on_channel before + * mgmt_tx or listen on a channel other than active one. */ - rc = wmi_mgmt_tx(vif, buf, len); - tx_status = (rc == 0); + if (params->chan && params->chan->hw_value == 0) { + wil_err(wil, "invalid channel\n"); + return -EINVAL; + } + if (wdev->iftype != NL80211_IFTYPE_AP) { + wil_dbg_misc(wil, + "send WMI_SW_TX_REQ_CMDID on non-AP interfaces\n"); + rc = wmi_mgmt_tx(vif, buf, len); + goto out; + } + + if (!params->chan || params->chan->hw_value == vif->channel) { + wil_dbg_misc(wil, + "send WMI_SW_TX_REQ_CMDID for on-channel\n"); + rc = wmi_mgmt_tx(vif, buf, len); + goto out; + } + + if (params->offchan == 0) { + wil_err(wil, + "invalid channel params: current %d requested %d, off-channel not allowed\n", + vif->channel, params->chan->hw_value); + return -EBUSY; + } + + /* use the wmi_mgmt_tx_ext only on AP mode and off-channel */ + rc = wmi_mgmt_tx_ext(vif, buf, len, params->chan->hw_value, + params->wait); + +out: + tx_status = (rc == 0); cfg80211_mgmt_tx_status(wdev, cookie ? *cookie : 0, buf, len, tx_status, GFP_KERNEL); + return rc; } diff --git a/drivers/net/wireless/ath/wil6210/debugfs.c b/drivers/net/wireless/ath/wil6210/debugfs.c index 58ce044b1130..51c3330bc316 100644 --- a/drivers/net/wireless/ath/wil6210/debugfs.c +++ b/drivers/net/wireless/ath/wil6210/debugfs.c @@ -1388,7 +1388,7 @@ static const struct file_operations fops_bf = { }; /*---------temp------------*/ -static void print_temp(struct seq_file *s, const char *prefix, u32 t) +static void print_temp(struct seq_file *s, const char *prefix, s32 t) { switch (t) { case 0: @@ -1396,7 +1396,8 @@ static void print_temp(struct seq_file *s, const char *prefix, u32 t) seq_printf(s, "%s N/A\n", prefix); break; default: - seq_printf(s, "%s %d.%03d\n", prefix, t / 1000, t % 1000); + seq_printf(s, "%s %s%d.%03d\n", prefix, (t < 0 ? "-" : ""), + abs(t / 1000), abs(t % 1000)); break; } } @@ -1404,7 +1405,7 @@ static void print_temp(struct seq_file *s, const char *prefix, u32 t) static int wil_temp_debugfs_show(struct seq_file *s, void *data) { struct wil6210_priv *wil = s->private; - u32 t_m, t_r; + s32 t_m, t_r; int rc = wmi_get_temperature(wil, &t_m, &t_r); if (rc) { @@ -1640,6 +1641,7 @@ static void wil_print_rxtid(struct seq_file *s, struct wil_tid_ampdu_rx *r) int i; u16 index = ((r->head_seq_num - r->ssn) & 0xfff) % r->buf_size; unsigned long long drop_dup = r->drop_dup, drop_old = r->drop_old; + unsigned long long drop_dup_mcast = r->drop_dup_mcast; seq_printf(s, "([%2d]) 0x%03x [", r->buf_size, r->head_seq_num); for (i = 0; i < r->buf_size; i++) { @@ -1649,9 +1651,9 @@ static void wil_print_rxtid(struct seq_file *s, struct wil_tid_ampdu_rx *r) seq_printf(s, "%c", r->reorder_buf[i] ? '*' : '_'); } seq_printf(s, - "] total %llu drop %llu (dup %llu + old %llu) last 0x%03x\n", - r->total, drop_dup + drop_old, drop_dup, drop_old, - r->ssn_last_drop); + "] total %llu drop %llu (dup %llu + old %llu + dup mcast %llu) last 0x%03x\n", + r->total, drop_dup + drop_old + drop_dup_mcast, drop_dup, + drop_old, drop_dup_mcast, r->ssn_last_drop); } static void wil_print_rxtid_crypto(struct seq_file *s, int tid, @@ -1733,13 +1735,12 @@ __acquires(&p->tid_rx_lock) __releases(&p->tid_rx_lock) p->stats.rx_short_frame, p->stats.rx_large_frame, p->stats.rx_replay); - - if (wil->use_enhanced_dma_hw) - seq_printf(s, - "mic error %lu, key error %lu, amsdu error %lu\n", - p->stats.rx_mic_error, - p->stats.rx_key_error, - p->stats.rx_amsdu_error); + seq_printf(s, + "mic error %lu, key error %lu, amsdu error %lu, csum error %lu\n", + p->stats.rx_mic_error, + p->stats.rx_key_error, + p->stats.rx_amsdu_error, + p->stats.rx_csum_err); seq_puts(s, "Rx/MCS:"); for (mcs = 0; mcs < ARRAY_SIZE(p->stats.rx_per_mcs); @@ -1801,6 +1802,343 @@ static const struct file_operations fops_mids = { .llseek = seq_lseek, }; +static int wil_tx_latency_debugfs_show(struct seq_file *s, void *data) +__acquires(&p->tid_rx_lock) __releases(&p->tid_rx_lock) +{ + struct wil6210_priv *wil = s->private; + int i, bin; + + for (i = 0; i < ARRAY_SIZE(wil->sta); i++) { + struct wil_sta_info *p = &wil->sta[i]; + char *status = "unknown"; + u8 aid = 0; + u8 mid; + + if (!p->tx_latency_bins) + continue; + + switch (p->status) { + case wil_sta_unused: + status = "unused "; + break; + case wil_sta_conn_pending: + status = "pending "; + break; + case wil_sta_connected: + status = "connected"; + aid = p->aid; + break; + } + mid = (p->status != wil_sta_unused) ? p->mid : U8_MAX; + seq_printf(s, "[%d] %pM %s MID %d AID %d\n", i, p->addr, status, + mid, aid); + + if (p->status == wil_sta_connected) { + u64 num_packets = 0; + u64 tx_latency_avg = p->stats.tx_latency_total_us; + + seq_puts(s, "Tx/Latency bin:"); + for (bin = 0; bin < WIL_NUM_LATENCY_BINS; bin++) { + seq_printf(s, " %lld", + p->tx_latency_bins[bin]); + num_packets += p->tx_latency_bins[bin]; + } + seq_puts(s, "\n"); + if (!num_packets) + continue; + do_div(tx_latency_avg, num_packets); + seq_printf(s, "Tx/Latency min/avg/max (us): %d/%lld/%d", + p->stats.tx_latency_min_us, + tx_latency_avg, + p->stats.tx_latency_max_us); + + seq_puts(s, "\n"); + } + } + + return 0; +} + +static int wil_tx_latency_seq_open(struct inode *inode, struct file *file) +{ + return single_open(file, wil_tx_latency_debugfs_show, + inode->i_private); +} + +static ssize_t wil_tx_latency_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct wil6210_priv *wil = s->private; + int val, rc, i; + bool enable; + + rc = kstrtoint_from_user(buf, len, 0, &val); + if (rc) { + wil_err(wil, "Invalid argument\n"); + return rc; + } + if (val == 1) + /* default resolution */ + val = 500; + if (val && (val < 50 || val > 1000)) { + wil_err(wil, "Invalid resolution %d\n", val); + return -EINVAL; + } + + enable = !!val; + if (wil->tx_latency == enable) + return len; + + wil_info(wil, "%s TX latency measurements (resolution %dusec)\n", + enable ? "Enabling" : "Disabling", val); + + if (enable) { + size_t sz = sizeof(u64) * WIL_NUM_LATENCY_BINS; + + wil->tx_latency_res = val; + for (i = 0; i < ARRAY_SIZE(wil->sta); i++) { + struct wil_sta_info *sta = &wil->sta[i]; + + kfree(sta->tx_latency_bins); + sta->tx_latency_bins = kzalloc(sz, GFP_KERNEL); + if (!sta->tx_latency_bins) + return -ENOMEM; + sta->stats.tx_latency_min_us = U32_MAX; + sta->stats.tx_latency_max_us = 0; + sta->stats.tx_latency_total_us = 0; + } + } + wil->tx_latency = enable; + + return len; +} + +static const struct file_operations fops_tx_latency = { + .open = wil_tx_latency_seq_open, + .release = single_release, + .read = seq_read, + .write = wil_tx_latency_write, + .llseek = seq_lseek, +}; + +static void wil_link_stats_print_basic(struct wil6210_vif *vif, + struct seq_file *s, + struct wmi_link_stats_basic *basic) +{ + char per[5] = "?"; + + if (basic->per_average != 0xff) + snprintf(per, sizeof(per), "%d%%", basic->per_average); + + seq_printf(s, "CID %d {\n" + "\tTxMCS %d TxTpt %d\n" + "\tGoodput(rx:tx) %d:%d\n" + "\tRxBcastFrames %d\n" + "\tRSSI %d SQI %d SNR %d PER %s\n" + "\tRx RFC %d Ant num %d\n" + "\tSectors(rx:tx) my %d:%d peer %d:%d\n" + "}\n", + basic->cid, + basic->bf_mcs, le32_to_cpu(basic->tx_tpt), + le32_to_cpu(basic->rx_goodput), + le32_to_cpu(basic->tx_goodput), + le32_to_cpu(basic->rx_bcast_frames), + basic->rssi, basic->sqi, basic->snr, per, + basic->selected_rfc, basic->rx_effective_ant_num, + basic->my_rx_sector, basic->my_tx_sector, + basic->other_rx_sector, basic->other_tx_sector); +} + +static void wil_link_stats_print_global(struct wil6210_priv *wil, + struct seq_file *s, + struct wmi_link_stats_global *global) +{ + seq_printf(s, "Frames(rx:tx) %d:%d\n" + "BA Frames(rx:tx) %d:%d\n" + "Beacons %d\n" + "Rx Errors (MIC:CRC) %d:%d\n" + "Tx Errors (no ack) %d\n", + le32_to_cpu(global->rx_frames), + le32_to_cpu(global->tx_frames), + le32_to_cpu(global->rx_ba_frames), + le32_to_cpu(global->tx_ba_frames), + le32_to_cpu(global->tx_beacons), + le32_to_cpu(global->rx_mic_errors), + le32_to_cpu(global->rx_crc_errors), + le32_to_cpu(global->tx_fail_no_ack)); +} + +static void wil_link_stats_debugfs_show_vif(struct wil6210_vif *vif, + struct seq_file *s) +{ + struct wil6210_priv *wil = vif_to_wil(vif); + struct wmi_link_stats_basic *stats; + int i; + + if (!vif->fw_stats_ready) { + seq_puts(s, "no statistics\n"); + return; + } + + seq_printf(s, "TSF %lld\n", vif->fw_stats_tsf); + for (i = 0; i < ARRAY_SIZE(wil->sta); i++) { + if (wil->sta[i].status == wil_sta_unused) + continue; + if (wil->sta[i].mid != vif->mid) + continue; + + stats = &wil->sta[i].fw_stats_basic; + wil_link_stats_print_basic(vif, s, stats); + } +} + +static int wil_link_stats_debugfs_show(struct seq_file *s, void *data) +{ + struct wil6210_priv *wil = s->private; + struct wil6210_vif *vif; + int i, rc; + + rc = mutex_lock_interruptible(&wil->vif_mutex); + if (rc) + return rc; + + /* iterate over all MIDs and show per-cid statistics. Then show the + * global statistics + */ + for (i = 0; i < wil->max_vifs; i++) { + vif = wil->vifs[i]; + + seq_printf(s, "MID %d ", i); + if (!vif) { + seq_puts(s, "unused\n"); + continue; + } + + wil_link_stats_debugfs_show_vif(vif, s); + } + + mutex_unlock(&wil->vif_mutex); + + return 0; +} + +static int wil_link_stats_seq_open(struct inode *inode, struct file *file) +{ + return single_open(file, wil_link_stats_debugfs_show, inode->i_private); +} + +static ssize_t wil_link_stats_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct wil6210_priv *wil = s->private; + int cid, interval, rc, i; + struct wil6210_vif *vif; + char *kbuf = kmalloc(len + 1, GFP_KERNEL); + + if (!kbuf) + return -ENOMEM; + + rc = simple_write_to_buffer(kbuf, len, ppos, buf, len); + if (rc != len) { + kfree(kbuf); + return rc >= 0 ? -EIO : rc; + } + + kbuf[len] = '\0'; + /* specify cid (use -1 for all cids) and snapshot interval in ms */ + rc = sscanf(kbuf, "%d %d", &cid, &interval); + kfree(kbuf); + if (rc < 0) + return rc; + if (rc < 2 || interval < 0) + return -EINVAL; + + wil_info(wil, "request link statistics, cid %d interval %d\n", + cid, interval); + + rc = mutex_lock_interruptible(&wil->vif_mutex); + if (rc) + return rc; + + for (i = 0; i < wil->max_vifs; i++) { + vif = wil->vifs[i]; + if (!vif) + continue; + + rc = wmi_link_stats_cfg(vif, WMI_LINK_STATS_TYPE_BASIC, + (cid == -1 ? 0xff : cid), interval); + if (rc) + wil_err(wil, "link statistics failed for mid %d\n", i); + } + mutex_unlock(&wil->vif_mutex); + + return len; +} + +static const struct file_operations fops_link_stats = { + .open = wil_link_stats_seq_open, + .release = single_release, + .read = seq_read, + .write = wil_link_stats_write, + .llseek = seq_lseek, +}; + +static int +wil_link_stats_global_debugfs_show(struct seq_file *s, void *data) +{ + struct wil6210_priv *wil = s->private; + + if (!wil->fw_stats_global.ready) + return 0; + + seq_printf(s, "TSF %lld\n", wil->fw_stats_global.tsf); + wil_link_stats_print_global(wil, s, &wil->fw_stats_global.stats); + + return 0; +} + +static int +wil_link_stats_global_seq_open(struct inode *inode, struct file *file) +{ + return single_open(file, wil_link_stats_global_debugfs_show, + inode->i_private); +} + +static ssize_t +wil_link_stats_global_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct wil6210_priv *wil = s->private; + int interval, rc; + struct wil6210_vif *vif = ndev_to_vif(wil->main_ndev); + + /* specify snapshot interval in ms */ + rc = kstrtoint_from_user(buf, len, 0, &interval); + if (rc || interval < 0) { + wil_err(wil, "Invalid argument\n"); + return -EINVAL; + } + + wil_info(wil, "request global link stats, interval %d\n", interval); + + rc = wmi_link_stats_cfg(vif, WMI_LINK_STATS_TYPE_GLOBAL, 0, interval); + if (rc) + wil_err(wil, "global link stats failed %d\n", rc); + + return rc ? rc : len; +} + +static const struct file_operations fops_link_stats_global = { + .open = wil_link_stats_global_seq_open, + .release = single_release, + .read = seq_read, + .write = wil_link_stats_global_write, + .llseek = seq_lseek, +}; + static ssize_t wil_read_file_led_cfg(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { @@ -2134,6 +2472,9 @@ static const struct { {"srings", 0444, &fops_srings}, {"status_msg", 0444, &fops_status_msg}, {"rx_buff_mgmt", 0444, &fops_rx_buff_mgmt}, + {"tx_latency", 0644, &fops_tx_latency}, + {"link_stats", 0644, &fops_link_stats}, + {"link_stats_global", 0644, &fops_link_stats_global}, }; static void wil6210_debugfs_init_files(struct wil6210_priv *wil, @@ -2250,10 +2591,14 @@ int wil6210_debugfs_init(struct wil6210_priv *wil) void wil6210_debugfs_remove(struct wil6210_priv *wil) { + int i; + debugfs_remove_recursive(wil->debug); wil->debug = NULL; kfree(wil->dbg_data.data_arr); + for (i = 0; i < ARRAY_SIZE(wil->sta); i++) + kfree(wil->sta[i].tx_latency_bins); /* free pmc memory without sending command to fw, as it will * be reset on the way down anyway diff --git a/drivers/net/wireless/ath/wil6210/fw.c b/drivers/net/wireless/ath/wil6210/fw.c index 540fc20984d8..3e2bbbceca06 100644 --- a/drivers/net/wireless/ath/wil6210/fw.c +++ b/drivers/net/wireless/ath/wil6210/fw.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2014-2015,2017 Qualcomm Atheros, Inc. + * Copyright (c) 2018, The Linux Foundation. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -22,6 +23,8 @@ MODULE_FIRMWARE(WIL_FW_NAME_DEFAULT); MODULE_FIRMWARE(WIL_FW_NAME_SPARROW_PLUS); MODULE_FIRMWARE(WIL_BOARD_FILE_NAME); +MODULE_FIRMWARE(WIL_FW_NAME_TALYN); +MODULE_FIRMWARE(WIL_BRD_NAME_TALYN); static void wil_memset_toio_32(volatile void __iomem *dst, u32 val, diff --git a/drivers/net/wireless/ath/wil6210/fw_inc.c b/drivers/net/wireless/ath/wil6210/fw_inc.c index 718161b829c2..388b3d4717ca 100644 --- a/drivers/net/wireless/ath/wil6210/fw_inc.c +++ b/drivers/net/wireless/ath/wil6210/fw_inc.c @@ -145,7 +145,7 @@ fw_handle_capabilities(struct wil6210_priv *wil, const void *data, capabilities); bitmap_zero(wil->fw_capabilities, WMI_FW_CAPABILITY_MAX); memcpy(wil->fw_capabilities, rec->capabilities, - min(sizeof(wil->fw_capabilities), capa_size)); + min_t(size_t, sizeof(wil->fw_capabilities), capa_size)); wil_hex_dump_fw("CAPA", DUMP_PREFIX_OFFSET, 16, 1, rec->capabilities, capa_size, false); return 0; diff --git a/drivers/net/wireless/ath/wil6210/interrupt.c b/drivers/net/wireless/ath/wil6210/interrupt.c index d7e112da6a8d..5d287a8e1b45 100644 --- a/drivers/net/wireless/ath/wil6210/interrupt.c +++ b/drivers/net/wireless/ath/wil6210/interrupt.c @@ -625,6 +625,15 @@ static irqreturn_t wil6210_irq_misc_thread(int irq, void *cookie) wil6210_unmask_irq_misc(wil, false); + /* in non-triple MSI case, this is done inside wil6210_thread_irq + * because it has to be done after unmasking the pseudo. + */ + if (wil->n_msi == 3 && wil->suspend_resp_rcvd) { + wil_dbg_irq(wil, "set suspend_resp_comp to true\n"); + wil->suspend_resp_comp = true; + wake_up_interruptible(&wil->wq); + } + return IRQ_HANDLED; } @@ -782,6 +791,40 @@ static irqreturn_t wil6210_hardirq(int irq, void *cookie) return rc; } +static int wil6210_request_3msi(struct wil6210_priv *wil, int irq) +{ + int rc; + + /* IRQ's are in the following order: + * - Tx + * - Rx + * - Misc + */ + rc = request_irq(irq, wil->txrx_ops.irq_tx, IRQF_SHARED, + WIL_NAME "_tx", wil); + if (rc) + return rc; + + rc = request_irq(irq + 1, wil->txrx_ops.irq_rx, IRQF_SHARED, + WIL_NAME "_rx", wil); + if (rc) + goto free0; + + rc = request_threaded_irq(irq + 2, wil6210_irq_misc, + wil6210_irq_misc_thread, + IRQF_SHARED, WIL_NAME "_misc", wil); + if (rc) + goto free1; + + return 0; +free1: + free_irq(irq + 1, wil); +free0: + free_irq(irq, wil); + + return rc; +} + /* can't use wil_ioread32_and_clear because ICC value is not set yet */ static inline void wil_clear32(void __iomem *addr) { @@ -822,11 +865,12 @@ void wil6210_clear_halp(struct wil6210_priv *wil) wil6210_unmask_halp(wil); } -int wil6210_init_irq(struct wil6210_priv *wil, int irq, bool use_msi) +int wil6210_init_irq(struct wil6210_priv *wil, int irq) { int rc; - wil_dbg_misc(wil, "init_irq: %s\n", use_msi ? "MSI" : "INTx"); + wil_dbg_misc(wil, "init_irq: %s, n_msi=%d\n", + wil->n_msi ? "MSI" : "INTx", wil->n_msi); if (wil->use_enhanced_dma_hw) { wil->txrx_ops.irq_tx = wil6210_irq_tx_edma; @@ -835,10 +879,14 @@ int wil6210_init_irq(struct wil6210_priv *wil, int irq, bool use_msi) wil->txrx_ops.irq_tx = wil6210_irq_tx; wil->txrx_ops.irq_rx = wil6210_irq_rx; } - rc = request_threaded_irq(irq, wil6210_hardirq, - wil6210_thread_irq, - use_msi ? 0 : IRQF_SHARED, - WIL_NAME, wil); + + if (wil->n_msi == 3) + rc = wil6210_request_3msi(wil, irq); + else + rc = request_threaded_irq(irq, wil6210_hardirq, + wil6210_thread_irq, + wil->n_msi ? 0 : IRQF_SHARED, + WIL_NAME, wil); return rc; } @@ -848,4 +896,8 @@ void wil6210_fini_irq(struct wil6210_priv *wil, int irq) wil_mask_irq(wil); free_irq(irq, wil); + if (wil->n_msi == 3) { + free_irq(irq + 1, wil); + free_irq(irq + 2, wil); + } } diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c index 4de19bd40a58..7debed6bec06 100644 --- a/drivers/net/wireless/ath/wil6210/main.c +++ b/drivers/net/wireless/ath/wil6210/main.c @@ -28,6 +28,7 @@ #define WAIT_FOR_HALP_VOTE_MS 100 #define WAIT_FOR_SCAN_ABORT_MS 1000 #define WIL_DEFAULT_NUM_RX_STATUS_RINGS 1 +#define WIL_BOARD_FILE_MAX_NAMELEN 128 bool debug_fw; /* = false; */ module_param(debug_fw, bool, 0444); @@ -278,6 +279,7 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock) } /* statistics */ memset(&sta->stats, 0, sizeof(sta->stats)); + sta->stats.tx_latency_min_us = U32_MAX; } static bool wil_vif_is_connected(struct wil6210_priv *wil, u8 mid) @@ -1128,6 +1130,9 @@ void wil_refresh_fw_capabilities(struct wil6210_priv *wil) wiphy->max_sched_scan_plans = WMI_MAX_PLANS_NUM; } + if (test_bit(WMI_FW_CAPABILITY_TX_REQ_EXT, wil->fw_capabilities)) + wiphy->flags |= WIPHY_FLAG_OFFCHAN_TX; + if (wil->platform_ops.set_features) { features = (test_bit(WMI_FW_CAPABILITY_REF_CLOCK_CONTROL, wil->fw_capabilities) && @@ -1135,8 +1140,20 @@ void wil_refresh_fw_capabilities(struct wil6210_priv *wil) wil->platform_capa)) ? BIT(WIL_PLATFORM_FEATURE_FW_EXT_CLK_CONTROL) : 0; + if (wil->n_msi == 3) + features |= BIT(WIL_PLATFORM_FEATURE_TRIPLE_MSI); + wil->platform_ops.set_features(wil->platform_handle, features); } + + if (test_bit(WMI_FW_CAPABILITY_BACK_WIN_SIZE_64, + wil->fw_capabilities)) { + wil->max_agg_wsize = WIL_MAX_AGG_WSIZE_64; + wil->max_ampdu_size = WIL_MAX_AMPDU_SIZE_128; + } else { + wil->max_agg_wsize = WIL_MAX_AGG_WSIZE; + wil->max_ampdu_size = WIL_MAX_AMPDU_SIZE; + } } void wil_mbox_ring_le2cpus(struct wil6210_mbox_ring *r) @@ -1148,6 +1165,28 @@ void wil_mbox_ring_le2cpus(struct wil6210_mbox_ring *r) le32_to_cpus(&r->head); } +/* construct actual board file name to use */ +void wil_get_board_file(struct wil6210_priv *wil, char *buf, size_t len) +{ + const char *board_file; + const char *wil_talyn_fw_name = ftm_mode ? WIL_FW_NAME_FTM_TALYN : + WIL_FW_NAME_TALYN; + + if (wil->board_file) { + board_file = wil->board_file; + } else { + /* If specific FW file is used for Talyn, + * use specific board file + */ + if (strcmp(wil->wil_fw_name, wil_talyn_fw_name) == 0) + board_file = WIL_BRD_NAME_TALYN; + else + board_file = WIL_BOARD_FILE_NAME; + } + + strlcpy(buf, board_file, len); +} + static int wil_get_bl_info(struct wil6210_priv *wil) { struct net_device *ndev = wil->main_ndev; @@ -1269,7 +1308,7 @@ static int wil_get_otp_info(struct wil6210_priv *wil) static int wil_wait_for_fw_ready(struct wil6210_priv *wil) { - ulong to = msecs_to_jiffies(1000); + ulong to = msecs_to_jiffies(2000); ulong left = wait_for_completion_timeout(&wil->wmi_ready, to); if (0 == left) { @@ -1519,8 +1558,17 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw) wil_set_oob_mode(wil, oob_mode); if (load_fw) { + char board_file[WIL_BOARD_FILE_MAX_NAMELEN]; + + if (wil->secured_boot) { + wil_err(wil, "secured boot is not supported\n"); + return -ENOTSUPP; + } + + board_file[0] = '\0'; + wil_get_board_file(wil, board_file, sizeof(board_file)); wil_info(wil, "Use firmware <%s> + board <%s>\n", - wil->wil_fw_name, WIL_BOARD_FILE_NAME); + wil->wil_fw_name, board_file); if (!no_flash) wil_bl_prepare_halt(wil); @@ -1532,11 +1580,9 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw) if (rc) goto out; if (wil->brd_file_addr) - rc = wil_request_board(wil, WIL_BOARD_FILE_NAME); + rc = wil_request_board(wil, board_file); else - rc = wil_request_firmware(wil, - WIL_BOARD_FILE_NAME, - true); + rc = wil_request_firmware(wil, board_file, true); if (rc) goto out; @@ -1568,6 +1614,13 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw) wil->txrx_ops.configure_interrupt_moderation(wil); + /* Enable OFU rdy valid bug fix, to prevent hang in oful34_rx + * while there is back-pressure from Host during RX + */ + if (wil->hw_version >= HW_VER_TALYN_MB) + wil_s(wil, RGF_DMA_MISC_CTL, + BIT_OFUL34_RDY_VALID_BUG_FIX_EN); + rc = wil_restore_vifs(wil); if (rc) { wil_err(wil, "failed to restore vifs, rc %d\n", rc); diff --git a/drivers/net/wireless/ath/wil6210/pcie_bus.c b/drivers/net/wireless/ath/wil6210/pcie_bus.c index 8b148cb91372..89119e7facd0 100644 --- a/drivers/net/wireless/ath/wil6210/pcie_bus.c +++ b/drivers/net/wireless/ath/wil6210/pcie_bus.c @@ -24,11 +24,11 @@ #include <linux/rtnetlink.h> #include <linux/pm_runtime.h> -static bool use_msi = true; -module_param(use_msi, bool, 0444); -MODULE_PARM_DESC(use_msi, " Use MSI interrupt, default - true"); +static int n_msi = 3; +module_param(n_msi, int, 0444); +MODULE_PARM_DESC(n_msi, " Use MSI interrupt: 0 - use INTx, 1 - single, or 3 - (default) "); -static bool ftm_mode; +bool ftm_mode; module_param(ftm_mode, bool, 0444); MODULE_PARM_DESC(ftm_mode, " Set factory test mode, default - false"); @@ -93,6 +93,10 @@ int wil_set_capabilities(struct wil6210_priv *wil) if (wil_r(wil, RGF_USER_OTP_HW_RD_MACHINE_1) & BIT_NO_FLASH_INDICATION) set_bit(hw_capa_no_flash, wil->hw_capa); + wil_fw_name = ftm_mode ? WIL_FW_NAME_FTM_TALYN : + WIL_FW_NAME_TALYN; + if (wil_fw_verify_file_exists(wil, wil_fw_name)) + wil->wil_fw_name = wil_fw_name; break; case JTAG_DEV_ID_TALYN_MB: wil->hw_name = "Talyn-MB"; @@ -104,6 +108,10 @@ int wil_set_capabilities(struct wil6210_priv *wil) set_bit(hw_capa_no_flash, wil->hw_capa); wil->use_enhanced_dma_hw = true; wil->use_rx_hw_reordering = true; + wil_fw_name = ftm_mode ? WIL_FW_NAME_FTM_TALYN : + WIL_FW_NAME_TALYN; + if (wil_fw_verify_file_exists(wil, wil_fw_name)) + wil->wil_fw_name = wil_fw_name; break; default: wil_err(wil, "Unknown board hardware, chip_id 0x%08x, chip_revision 0x%08x\n", @@ -142,12 +150,24 @@ int wil_set_capabilities(struct wil6210_priv *wil) void wil_disable_irq(struct wil6210_priv *wil) { - disable_irq(wil->pdev->irq); + int irq = wil->pdev->irq; + + disable_irq(irq); + if (wil->n_msi == 3) { + disable_irq(irq + 1); + disable_irq(irq + 2); + } } void wil_enable_irq(struct wil6210_priv *wil) { - enable_irq(wil->pdev->irq); + int irq = wil->pdev->irq; + + enable_irq(irq); + if (wil->n_msi == 3) { + enable_irq(irq + 1); + enable_irq(irq + 2); + } } static void wil_remove_all_additional_vifs(struct wil6210_priv *wil) @@ -174,28 +194,47 @@ static int wil_if_pcie_enable(struct wil6210_priv *wil) * and only MSI should be used */ int msi_only = pdev->msi_enabled; - bool _use_msi = use_msi; wil_dbg_misc(wil, "if_pcie_enable\n"); pci_set_master(pdev); - wil_dbg_misc(wil, "Setup %s interrupt\n", use_msi ? "MSI" : "INTx"); + /* how many MSI interrupts to request? */ + switch (n_msi) { + case 3: + case 1: + wil_dbg_misc(wil, "Setup %d MSI interrupts\n", n_msi); + break; + case 0: + wil_dbg_misc(wil, "MSI interrupts disabled, use INTx\n"); + break; + default: + wil_err(wil, "Invalid n_msi=%d, default to 1\n", n_msi); + n_msi = 1; + } + + if (n_msi == 3 && + pci_alloc_irq_vectors(pdev, n_msi, n_msi, PCI_IRQ_MSI) < n_msi) { + wil_err(wil, "3 MSI mode failed, try 1 MSI\n"); + n_msi = 1; + } - if (use_msi && pci_enable_msi(pdev)) { + if (n_msi == 1 && pci_enable_msi(pdev)) { wil_err(wil, "pci_enable_msi failed, use INTx\n"); - _use_msi = false; + n_msi = 0; } - if (!_use_msi && msi_only) { + wil->n_msi = n_msi; + + if (wil->n_msi == 0 && msi_only) { wil_err(wil, "Interrupt pin not routed, unable to use INTx\n"); rc = -ENODEV; goto stop_master; } - rc = wil6210_init_irq(wil, pdev->irq, _use_msi); + rc = wil6210_init_irq(wil, pdev->irq); if (rc) - goto stop_master; + goto release_vectors; /* need reset here to obtain MAC */ mutex_lock(&wil->mutex); @@ -208,8 +247,9 @@ static int wil_if_pcie_enable(struct wil6210_priv *wil) release_irq: wil6210_fini_irq(wil, pdev->irq); - /* safe to call if no MSI */ - pci_disable_msi(pdev); + release_vectors: + /* safe to call if no allocation */ + pci_free_irq_vectors(pdev); stop_master: pci_clear_master(pdev); return rc; diff --git a/drivers/net/wireless/ath/wil6210/rx_reorder.c b/drivers/net/wireless/ath/wil6210/rx_reorder.c index 22475a1ddb7f..b608aa16b4f1 100644 --- a/drivers/net/wireless/ath/wil6210/rx_reorder.c +++ b/drivers/net/wireless/ath/wil6210/rx_reorder.c @@ -95,7 +95,7 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock) { struct wil6210_vif *vif; struct net_device *ndev; - int tid, cid, mid, mcast; + int tid, cid, mid, mcast, retry; u16 seq; struct wil_sta_info *sta; struct wil_tid_ampdu_rx *r; @@ -103,7 +103,7 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock) int index; wil->txrx_ops.get_reorder_params(wil, skb, &tid, &cid, &mid, &seq, - &mcast); + &mcast, &retry); sta = &wil->sta[cid]; wil_dbg_txrx(wil, "MID %d CID %d TID %d Seq 0x%03x mcast %01x\n", @@ -117,11 +117,6 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock) } ndev = vif_to_ndev(vif); - if (unlikely(mcast)) { - wil_netif_rx_any(skb, ndev); - return; - } - spin_lock(&sta->tid_rx_lock); r = sta->tid_rx[tid]; @@ -130,6 +125,19 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock) goto out; } + if (unlikely(mcast)) { + if (retry && seq == r->mcast_last_seq) { + r->drop_dup_mcast++; + wil_dbg_txrx(wil, "Rx drop: dup mcast seq 0x%03x\n", + seq); + dev_kfree_skb(skb); + goto out; + } + r->mcast_last_seq = seq; + wil_netif_rx_any(skb, ndev); + goto out; + } + r->total++; hseq = r->head_seq_num; @@ -262,6 +270,7 @@ struct wil_tid_ampdu_rx *wil_tid_ampdu_rx_alloc(struct wil6210_priv *wil, r->buf_size = size; r->stored_mpdu_num = 0; r->first_time = true; + r->mcast_last_seq = U16_MAX; return r; } @@ -288,7 +297,7 @@ void wil_tid_ampdu_rx_free(struct wil6210_priv *wil, /* ADDBA processing */ static u16 wil_agg_size(struct wil6210_priv *wil, u16 req_agg_wsize) { - u16 max_agg_size = min_t(u16, WIL_MAX_AGG_WSIZE, WIL_MAX_AMPDU_SIZE / + u16 max_agg_size = min_t(u16, wil->max_agg_wsize, wil->max_ampdu_size / (mtu_max + WIL_MAX_MPDU_OVERHEAD)); if (!req_agg_wsize) @@ -355,11 +364,11 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock) if (status == WLAN_STATUS_SUCCESS) { if (req_agg_wsize == 0) { wil_dbg_misc(wil, "Suggest BACK wsize %d\n", - WIL_MAX_AGG_WSIZE); - agg_wsize = WIL_MAX_AGG_WSIZE; + wil->max_agg_wsize); + agg_wsize = wil->max_agg_wsize; } else { agg_wsize = min_t(u16, - WIL_MAX_AGG_WSIZE, req_agg_wsize); + wil->max_agg_wsize, req_agg_wsize); } } diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c index 2098f3cc1cec..6a7943e487fb 100644 --- a/drivers/net/wireless/ath/wil6210/txrx.c +++ b/drivers/net/wireless/ath/wil6210/txrx.c @@ -281,6 +281,12 @@ static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring, skb_reserve(skb, headroom); skb_put(skb, sz); + /** + * Make sure that the network stack calculates checksum for packets + * which failed the HW checksum calculation + */ + skb->ip_summed = CHECKSUM_NONE; + pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(dev, pa))) { kfree_skb(skb); @@ -569,6 +575,8 @@ again: * mis-calculates TCP checksum - if it should be 0x0, * it writes 0xffff in violation of RFC 1624 */ + else + stats->rx_csum_err++; } if (snaplen) { @@ -678,6 +686,21 @@ static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb) return 0; } +static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb, + struct wil_net_stats *stats) +{ + struct vring_rx_desc *d = wil_skb_rxdesc(skb); + + if ((d->dma.status & RX_DMA_STATUS_ERROR) && + (d->dma.error & RX_DMA_ERROR_MIC)) { + stats->rx_mic_error++; + wil_dbg_txrx(wil, "MIC error, dropping packet\n"); + return -EFAULT; + } + + return 0; +} + static void wil_get_netif_rx_params(struct sk_buff *skb, int *cid, int *security) { @@ -736,6 +759,12 @@ void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev) goto stats; } + /* check errors reported by HW and update statistics */ + if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) { + dev_kfree_skb(skb); + return; + } + if (wdev->iftype == NL80211_IFTYPE_AP && !vif->ap_isolate) { if (mcast) { /* send multicast frames both to higher layers in @@ -1672,6 +1701,11 @@ static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif, */ wmb(); + if (wil->tx_latency) + *(ktime_t *)&skb->cb = ktime_get(); + else + memset(skb->cb, 0, sizeof(ktime_t)); + wil_w(wil, vring->hwtail, vring->swhead); return 0; @@ -1823,6 +1857,11 @@ static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif, */ wmb(); + if (wil->tx_latency) + *(ktime_t *)&skb->cb = ktime_get(); + else + memset(skb->cb, 0, sizeof(ktime_t)); + wil_w(wil, ring->hwtail, ring->swhead); return 0; @@ -2044,6 +2083,31 @@ netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev) return NET_XMIT_DROP; } +void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb, + struct wil_sta_info *sta) +{ + int skb_time_us; + int bin; + + if (!wil->tx_latency) + return; + + if (ktime_to_ms(*(ktime_t *)&skb->cb) == 0) + return; + + skb_time_us = ktime_us_delta(ktime_get(), *(ktime_t *)&skb->cb); + bin = skb_time_us / wil->tx_latency_res; + bin = min_t(int, bin, WIL_NUM_LATENCY_BINS - 1); + + wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin); + sta->tx_latency_bins[bin]++; + sta->stats.tx_latency_total_us += skb_time_us; + if (skb_time_us < sta->stats.tx_latency_min_us) + sta->stats.tx_latency_min_us = skb_time_us; + if (skb_time_us > sta->stats.tx_latency_max_us) + sta->stats.tx_latency_max_us = skb_time_us; +} + /** * Clean up transmitted skb's from the Tx VRING * @@ -2130,6 +2194,9 @@ int wil_tx_complete(struct wil6210_vif *vif, int ringid) if (stats) { stats->tx_packets++; stats->tx_bytes += skb->len; + + wil_tx_latency_calc(wil, skb, + &wil->sta[cid]); } } else { ndev->stats.tx_errors++; @@ -2180,7 +2247,7 @@ static inline void wil_tx_fini(struct wil6210_priv *wil) {} static void wil_get_reorder_params(struct wil6210_priv *wil, struct sk_buff *skb, int *tid, int *cid, - int *mid, u16 *seq, int *mcast) + int *mid, u16 *seq, int *mcast, int *retry) { struct vring_rx_desc *d = wil_skb_rxdesc(skb); @@ -2189,6 +2256,7 @@ static void wil_get_reorder_params(struct wil6210_priv *wil, *mid = wil_rxdesc_mid(d); *seq = wil_rxdesc_seq(d); *mcast = wil_rxdesc_mcast(d); + *retry = wil_rxdesc_retry(d); } void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil) @@ -2211,6 +2279,7 @@ void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil) wil->txrx_ops.get_netif_rx_params = wil_get_netif_rx_params; wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check; + wil->txrx_ops.rx_error_check = wil_rx_error_check; wil->txrx_ops.is_rx_idle = wil_is_rx_idle; wil->txrx_ops.rx_fini = wil_rx_fini; } diff --git a/drivers/net/wireless/ath/wil6210/txrx.h b/drivers/net/wireless/ath/wil6210/txrx.h index f361423628f5..9d83be481839 100644 --- a/drivers/net/wireless/ath/wil6210/txrx.h +++ b/drivers/net/wireless/ath/wil6210/txrx.h @@ -500,6 +500,11 @@ static inline int wil_rxdesc_ext_subtype(struct vring_rx_desc *d) return WIL_GET_BITS(d->mac.d0, 28, 31); } +static inline int wil_rxdesc_retry(struct vring_rx_desc *d) +{ + return WIL_GET_BITS(d->mac.d0, 31, 31); +} + static inline int wil_rxdesc_key_id(struct vring_rx_desc *d) { return WIL_GET_BITS(d->mac.d1, 4, 5); @@ -615,5 +620,7 @@ void wil_tid_ampdu_rx_free(struct wil6210_priv *wil, struct wil_tid_ampdu_rx *r); void wil_tx_data_init(struct wil_ring_tx_data *txdata); void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil); +void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb, + struct wil_sta_info *sta); #endif /* WIL6210_TXRX_H */ diff --git a/drivers/net/wireless/ath/wil6210/txrx_edma.c b/drivers/net/wireless/ath/wil6210/txrx_edma.c index 95f38e65d969..bca61cb44c37 100644 --- a/drivers/net/wireless/ath/wil6210/txrx_edma.c +++ b/drivers/net/wireless/ath/wil6210/txrx_edma.c @@ -27,6 +27,8 @@ #include "trace.h" #define WIL_EDMA_MAX_DATA_OFFSET (2) +/* RX buffer size must be aligned to 4 bytes */ +#define WIL_EDMA_RX_BUF_LEN_DEFAULT (2048) static void wil_tx_desc_unmap_edma(struct device *dev, union wil_tx_desc *desc, @@ -158,8 +160,7 @@ static int wil_ring_alloc_skb_edma(struct wil6210_priv *wil, struct wil_ring *ring, u32 i) { struct device *dev = wil_to_dev(wil); - unsigned int sz = wil->rx_buf_len + ETH_HLEN + - WIL_EDMA_MAX_DATA_OFFSET; + unsigned int sz = ALIGN(wil->rx_buf_len, 4); dma_addr_t pa; u16 buff_id; struct list_head *active = &wil->rx_buff_mgmt.active; @@ -182,6 +183,12 @@ static int wil_ring_alloc_skb_edma(struct wil6210_priv *wil, skb_put(skb, sz); + /** + * Make sure that the network stack calculates checksum for packets + * which failed the HW checksum calculation + */ + skb->ip_summed = CHECKSUM_NONE; + pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(dev, pa))) { kfree_skb(skb); @@ -503,7 +510,7 @@ out_free: static void wil_get_reorder_params_edma(struct wil6210_priv *wil, struct sk_buff *skb, int *tid, int *cid, int *mid, u16 *seq, - int *mcast) + int *mcast, int *retry) { struct wil_rx_status_extended *s = wil_skb_rxstatus(skb); @@ -512,6 +519,7 @@ static void wil_get_reorder_params_edma(struct wil6210_priv *wil, *mid = wil_rx_status_get_mid(s); *seq = le16_to_cpu(wil_rx_status_get_seq(wil, s)); *mcast = wil_rx_status_get_mcast(s); + *retry = wil_rx_status_get_retry(s); } static void wil_get_netif_rx_params_edma(struct sk_buff *skb, int *cid, @@ -593,7 +601,7 @@ static bool wil_is_rx_idle_edma(struct wil6210_priv *wil) static void wil_rx_buf_len_init_edma(struct wil6210_priv *wil) { wil->rx_buf_len = rx_large_buf ? - WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD; + WIL_MAX_ETH_MTU : WIL_EDMA_RX_BUF_LEN_DEFAULT; } static int wil_rx_init_edma(struct wil6210_priv *wil, u16 desc_ring_size) @@ -626,8 +634,7 @@ static int wil_rx_init_edma(struct wil6210_priv *wil, u16 desc_ring_size) wil_rx_buf_len_init_edma(wil); - max_rx_pl_per_desc = wil->rx_buf_len + ETH_HLEN + - WIL_EDMA_MAX_DATA_OFFSET; + max_rx_pl_per_desc = ALIGN(wil->rx_buf_len, 4); /* Use debugfs dbg_num_rx_srings if set, reserve one sring for TX */ if (wil->num_rx_status_rings > WIL6210_MAX_STATUS_RINGS - 1) @@ -794,14 +801,15 @@ static int wil_check_bar(struct wil6210_priv *wil, void *msg, int cid, return -EAGAIN; } -static int wil_rx_edma_check_errors(struct wil6210_priv *wil, void *msg, - struct wil_net_stats *stats, - struct sk_buff *skb) +static int wil_rx_error_check_edma(struct wil6210_priv *wil, + struct sk_buff *skb, + struct wil_net_stats *stats) { int error; int l2_rx_status; int l3_rx_status; int l4_rx_status; + void *msg = wil_skb_rxstatus(skb); error = wil_rx_status_get_error(msg); if (!error) { @@ -845,6 +853,8 @@ static int wil_rx_edma_check_errors(struct wil6210_priv *wil, void *msg, * mis-calculates TCP checksum - if it should be 0x0, * it writes 0xffff in violation of RFC 1624 */ + else + stats->rx_csum_err++; return 0; } @@ -859,12 +869,10 @@ static struct sk_buff *wil_sring_reap_rx_edma(struct wil6210_priv *wil, struct sk_buff *skb; dma_addr_t pa; struct wil_ring_rx_data *rxdata = &sring->rx_data; - unsigned int sz = wil->rx_buf_len + ETH_HLEN + - WIL_EDMA_MAX_DATA_OFFSET; + unsigned int sz = ALIGN(wil->rx_buf_len, 4); struct wil_net_stats *stats = NULL; u16 dmalen; int cid; - int rc; bool eop, headstolen; int delta; u8 dr_bit; @@ -936,13 +944,6 @@ again: goto skipping; } - /* Check and treat errors reported by HW */ - rc = wil_rx_edma_check_errors(wil, msg, stats, skb); - if (rc) { - rxdata->skipping = true; - goto skipping; - } - if (unlikely(dmalen > sz)) { wil_err(wil, "Rx size too large: %d bytes!\n", dmalen); stats->rx_large_frame++; @@ -1223,6 +1224,9 @@ int wil_tx_sring_handler(struct wil6210_priv *wil, if (stats) { stats->tx_packets++; stats->tx_bytes += skb->len; + + wil_tx_latency_calc(wil, skb, + &wil->sta[cid]); } } else { ndev->stats.tx_errors++; @@ -1473,6 +1477,11 @@ static int __wil_tx_ring_tso_edma(struct wil6210_priv *wil, */ wmb(); + if (wil->tx_latency) + *(ktime_t *)&skb->cb = ktime_get(); + else + memset(skb->cb, 0, sizeof(ktime_t)); + wil_w(wil, ring->hwtail, ring->swhead); return 0; @@ -1592,6 +1601,7 @@ void wil_init_txrx_ops_edma(struct wil6210_priv *wil) wil->txrx_ops.get_reorder_params = wil_get_reorder_params_edma; wil->txrx_ops.get_netif_rx_params = wil_get_netif_rx_params_edma; wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check_edma; + wil->txrx_ops.rx_error_check = wil_rx_error_check_edma; wil->txrx_ops.is_rx_idle = wil_is_rx_idle_edma; wil->txrx_ops.rx_fini = wil_rx_fini_edma; } diff --git a/drivers/net/wireless/ath/wil6210/txrx_edma.h b/drivers/net/wireless/ath/wil6210/txrx_edma.h index e86fc2dc0ce0..a7fe9292fda3 100644 --- a/drivers/net/wireless/ath/wil6210/txrx_edma.h +++ b/drivers/net/wireless/ath/wil6210/txrx_edma.h @@ -471,6 +471,12 @@ static inline __le16 wil_rx_status_get_seq(struct wil6210_priv *wil, void *msg) return ((struct wil_rx_status_extended *)msg)->ext.seq_num; } +static inline u8 wil_rx_status_get_retry(void *msg) +{ + /* retry bit is missing in EDMA HW. return 1 to be on the safe side */ + return 1; +} + static inline int wil_rx_status_get_mid(void *msg) { if (!(((struct wil_rx_status_compressed *)msg)->d0 & diff --git a/drivers/net/wireless/ath/wil6210/wil6210.h b/drivers/net/wireless/ath/wil6210/wil6210.h index d963c76b679e..17c294b1ead1 100644 --- a/drivers/net/wireless/ath/wil6210/wil6210.h +++ b/drivers/net/wireless/ath/wil6210/wil6210.h @@ -37,6 +37,7 @@ extern bool rx_align_2; extern bool rx_large_buf; extern bool debug_fw; extern bool disable_ap_sme; +extern bool ftm_mode; struct wil6210_priv; struct wil6210_vif; @@ -50,11 +51,17 @@ union wil_tx_desc; #define WIL_FW_NAME_SPARROW_PLUS "wil6210_sparrow_plus.fw" #define WIL_FW_NAME_FTM_SPARROW_PLUS "wil6210_sparrow_plus_ftm.fw" +#define WIL_FW_NAME_TALYN "wil6436.fw" +#define WIL_FW_NAME_FTM_TALYN "wil6436_ftm.fw" +#define WIL_BRD_NAME_TALYN "wil6436.brd" + #define WIL_BOARD_FILE_NAME "wil6210.brd" /* board & radio parameters */ #define WIL_DEFAULT_BUS_REQUEST_KBPS 128000 /* ~1Gbps */ #define WIL_MAX_BUS_REQUEST_KBPS 800000 /* ~6.1Gbps */ +#define WIL_NUM_LATENCY_BINS 200 + /* maximum number of virtual interfaces the driver supports * (including the main interface) */ @@ -85,6 +92,8 @@ static inline u32 WIL_GET_BITS(u32 x, int b0, int b1) #define WIL6210_NAPI_BUDGET (16) /* arbitrary */ #define WIL_MAX_AMPDU_SIZE (64 * 1024) /* FW/HW limit */ #define WIL_MAX_AGG_WSIZE (32) /* FW/HW limit */ +#define WIL_MAX_AMPDU_SIZE_128 (128 * 1024) /* FW/HW limit */ +#define WIL_MAX_AGG_WSIZE_64 (64) /* FW/HW limit */ #define WIL6210_MAX_STATUS_RINGS (8) /* Hardware offload block adds the following: @@ -293,6 +302,8 @@ struct RGF_ICR { #define BIT_DMA_ITR_RX_IDL_CNT_CTL_FOREVER BIT(2) #define BIT_DMA_ITR_RX_IDL_CNT_CTL_CLR BIT(3) #define BIT_DMA_ITR_RX_IDL_CNT_CTL_REACHED_TRESH BIT(4) +#define RGF_DMA_MISC_CTL (0x881d6c) + #define BIT_OFUL34_RDY_VALID_BUG_FIX_EN BIT(7) #define RGF_DMA_PSEUDO_CAUSE (0x881c68) #define RGF_DMA_PSEUDO_CAUSE_MASK_SW (0x881c6c) @@ -543,6 +554,31 @@ struct wil_status_ring { struct wil_ring_rx_data rx_data; }; +#define WIL_STA_TID_NUM (16) +#define WIL_MCS_MAX (12) /* Maximum MCS supported */ + +struct wil_net_stats { + unsigned long rx_packets; + unsigned long tx_packets; + unsigned long rx_bytes; + unsigned long tx_bytes; + unsigned long tx_errors; + u32 tx_latency_min_us; + u32 tx_latency_max_us; + u64 tx_latency_total_us; + unsigned long rx_dropped; + unsigned long rx_non_data_frame; + unsigned long rx_short_frame; + unsigned long rx_large_frame; + unsigned long rx_replay; + unsigned long rx_mic_error; + unsigned long rx_key_error; /* eDMA specific */ + unsigned long rx_amsdu_error; /* eDMA specific */ + unsigned long rx_csum_err; + u16 last_mcs_rx; + u64 rx_per_mcs[WIL_MCS_MAX + 1]; +}; + /** * struct tx_rx_ops - different TX/RX ops for legacy and enhanced * DMA flow @@ -572,10 +608,12 @@ struct wil_txrx_ops { u16 agg_wsize, u16 timeout); void (*get_reorder_params)(struct wil6210_priv *wil, struct sk_buff *skb, int *tid, int *cid, - int *mid, u16 *seq, int *mcast); + int *mid, u16 *seq, int *mcast, int *retry); void (*get_netif_rx_params)(struct sk_buff *skb, int *cid, int *security); int (*rx_crypto_check)(struct wil6210_priv *wil, struct sk_buff *skb); + int (*rx_error_check)(struct wil6210_priv *wil, struct sk_buff *skb, + struct wil_net_stats *stats); bool (*is_rx_idle)(struct wil6210_priv *wil); irqreturn_t (*irq_rx)(int irq, void *cookie); }; @@ -625,6 +663,8 @@ struct pci_dev; * @drop_dup: duplicate frames dropped for this reorder buffer * @drop_old: old frames dropped for this reorder buffer * @first_time: true when this buffer used 1-st time + * @mcast_last_seq: sequence number (SN) of last received multicast packet + * @drop_dup_mcast: duplicate multicast frames dropped for this reorder buffer */ struct wil_tid_ampdu_rx { struct sk_buff **reorder_buf; @@ -638,6 +678,8 @@ struct wil_tid_ampdu_rx { unsigned long long drop_dup; unsigned long long drop_old; bool first_time; /* is it 1-st time this buffer used? */ + u16 mcast_last_seq; /* multicast dup detection */ + unsigned long long drop_dup_mcast; }; /** @@ -672,27 +714,6 @@ enum wil_sta_status { wil_sta_connected = 2, }; -#define WIL_STA_TID_NUM (16) -#define WIL_MCS_MAX (12) /* Maximum MCS supported */ - -struct wil_net_stats { - unsigned long rx_packets; - unsigned long tx_packets; - unsigned long rx_bytes; - unsigned long tx_bytes; - unsigned long tx_errors; - unsigned long rx_dropped; - unsigned long rx_non_data_frame; - unsigned long rx_short_frame; - unsigned long rx_large_frame; - unsigned long rx_replay; - unsigned long rx_mic_error; /* eDMA specific */ - unsigned long rx_key_error; /* eDMA specific */ - unsigned long rx_amsdu_error; /* eDMA specific */ - u16 last_mcs_rx; - u64 rx_per_mcs[WIL_MCS_MAX + 1]; -}; - /** * struct wil_sta_info - data for peer * @@ -706,6 +727,14 @@ struct wil_sta_info { u8 mid; enum wil_sta_status status; struct wil_net_stats stats; + /** + * 20 latency bins. 1st bin counts packets with latency + * of 0..tx_latency_res, last bin counts packets with latency + * of 19*tx_latency_res and above. + * tx_latency_res is configured from "tx_latency" debug-fs. + */ + u64 *tx_latency_bins; + struct wmi_link_stats_basic fw_stats_basic; /* Rx BACK */ struct wil_tid_ampdu_rx *tid_rx[WIL_STA_TID_NUM]; spinlock_t tid_rx_lock; /* guarding tid_rx array */ @@ -820,6 +849,8 @@ struct wil6210_vif { struct mutex probe_client_mutex; /* protect @probe_client_pending */ struct work_struct probe_client_worker; int net_queue_stopped; /* netif_tx_stop_all_queues invoked */ + bool fw_stats_ready; /* per-cid statistics are ready inside sta_info */ + u64 fw_stats_tsf; /* measurement timestamp */ }; /** @@ -847,11 +878,18 @@ struct wil_rx_buff_mgmt { unsigned long free_list_empty_cnt; /* statistics */ }; +struct wil_fw_stats_global { + bool ready; + u64 tsf; /* measurement timestamp */ + struct wmi_link_stats_global stats; +}; + struct wil6210_priv { struct pci_dev *pdev; u32 bar_size; struct wiphy *wiphy; struct net_device *main_ndev; + int n_msi; void __iomem *csr; DECLARE_BITMAP(status, wil_status_last); u8 fw_version[ETHTOOL_FWVERS_LEN]; @@ -937,6 +975,8 @@ struct wil6210_priv { u8 wakeup_trigger; struct wil_suspend_stats suspend_stats; struct wil_debugfs_data dbg_data; + bool tx_latency; /* collect TX latency measurements */ + size_t tx_latency_res; /* bin resolution in usec */ void *platform_handle; struct wil_platform_ops platform_ops; @@ -977,6 +1017,11 @@ struct wil6210_priv { bool use_rx_hw_reordering; bool secured_boot; u8 boot_config; + + struct wil_fw_stats_global fw_stats_global; + + u32 max_agg_wsize; + u32 max_ampdu_size; }; #define wil_to_wiphy(i) (i->wiphy) @@ -1060,6 +1105,8 @@ static inline void wil_c(struct wil6210_priv *wil, u32 reg, u32 val) wil_w(wil, reg, wil_r(wil, reg) & ~val); } +void wil_get_board_file(struct wil6210_priv *wil, char *buf, size_t len); + #if defined(CONFIG_DYNAMIC_DEBUG) #define wil_hex_dump_txrx(prefix_str, prefix_type, rowsize, \ groupsize, buf, len, ascii) \ @@ -1176,13 +1223,14 @@ int wmi_new_sta(struct wil6210_vif *vif, const u8 *mac, u8 aid); int wmi_port_allocate(struct wil6210_priv *wil, u8 mid, const u8 *mac, enum nl80211_iftype iftype); int wmi_port_delete(struct wil6210_priv *wil, u8 mid); +int wmi_link_stats_cfg(struct wil6210_vif *vif, u32 type, u8 cid, u32 interval); int wil_addba_rx_request(struct wil6210_priv *wil, u8 mid, u8 cidxtid, u8 dialog_token, __le16 ba_param_set, __le16 ba_timeout, __le16 ba_seq_ctrl); int wil_addba_tx_request(struct wil6210_priv *wil, u8 ringid, u16 wsize); void wil6210_clear_irq(struct wil6210_priv *wil); -int wil6210_init_irq(struct wil6210_priv *wil, int irq, bool use_msi); +int wil6210_init_irq(struct wil6210_priv *wil, int irq); void wil6210_fini_irq(struct wil6210_priv *wil, int irq); void wil_mask_irq(struct wil6210_priv *wil); void wil_unmask_irq(struct wil6210_priv *wil); @@ -1304,6 +1352,8 @@ int wmi_start_sched_scan(struct wil6210_priv *wil, struct cfg80211_sched_scan_request *request); int wmi_stop_sched_scan(struct wil6210_priv *wil); int wmi_mgmt_tx(struct wil6210_vif *vif, const u8 *buf, size_t len); +int wmi_mgmt_tx_ext(struct wil6210_vif *vif, const u8 *buf, size_t len, + u8 channel, u16 duration_ms); int reverse_memcmp(const void *cs, const void *ct, size_t count); diff --git a/drivers/net/wireless/ath/wil6210/wil_platform.h b/drivers/net/wireless/ath/wil6210/wil_platform.h index 177026e5323b..bca090611477 100644 --- a/drivers/net/wireless/ath/wil6210/wil_platform.h +++ b/drivers/net/wireless/ath/wil6210/wil_platform.h @@ -29,6 +29,7 @@ enum wil_platform_event { enum wil_platform_features { WIL_PLATFORM_FEATURE_FW_EXT_CLK_CONTROL = 0, + WIL_PLATFORM_FEATURE_TRIPLE_MSI = 1, WIL_PLATFORM_FEATURE_MAX, }; diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c index 71056c834fff..42c02a20ec97 100644 --- a/drivers/net/wireless/ath/wil6210/wmi.c +++ b/drivers/net/wireless/ath/wil6210/wmi.c @@ -464,6 +464,10 @@ static const char *cmdid2name(u16 cmdid) return "WMI_BCAST_DESC_RING_ADD_CMD"; case WMI_CFG_DEF_RX_OFFLOAD_CMDID: return "WMI_CFG_DEF_RX_OFFLOAD_CMD"; + case WMI_LINK_STATS_CMDID: + return "WMI_LINK_STATS_CMD"; + case WMI_SW_TX_REQ_EXT_CMDID: + return "WMI_SW_TX_REQ_EXT_CMDID"; default: return "Untracked CMD"; } @@ -598,6 +602,10 @@ static const char *eventid2name(u16 eventid) return "WMI_RX_DESC_RING_CFG_DONE_EVENT"; case WMI_CFG_DEF_RX_OFFLOAD_DONE_EVENTID: return "WMI_CFG_DEF_RX_OFFLOAD_DONE_EVENT"; + case WMI_LINK_STATS_CONFIG_DONE_EVENTID: + return "WMI_LINK_STATS_CONFIG_DONE_EVENT"; + case WMI_LINK_STATS_EVENTID: + return "WMI_LINK_STATS_EVENT"; default: return "Untracked EVENT"; } @@ -1329,6 +1337,130 @@ wmi_evt_sched_scan_result(struct wil6210_vif *vif, int id, void *d, int len) cfg80211_sched_scan_results(wiphy, 0); } +static void wil_link_stats_store_basic(struct wil6210_vif *vif, + struct wmi_link_stats_basic *basic) +{ + struct wil6210_priv *wil = vif_to_wil(vif); + u8 cid = basic->cid; + struct wil_sta_info *sta; + + if (cid < 0 || cid >= WIL6210_MAX_CID) { + wil_err(wil, "invalid cid %d\n", cid); + return; + } + + sta = &wil->sta[cid]; + sta->fw_stats_basic = *basic; +} + +static void wil_link_stats_store_global(struct wil6210_vif *vif, + struct wmi_link_stats_global *global) +{ + struct wil6210_priv *wil = vif_to_wil(vif); + + wil->fw_stats_global.stats = *global; +} + +static void wmi_link_stats_parse(struct wil6210_vif *vif, u64 tsf, + bool has_next, void *payload, + size_t payload_size) +{ + struct wil6210_priv *wil = vif_to_wil(vif); + size_t hdr_size = sizeof(struct wmi_link_stats_record); + size_t stats_size, record_size, expected_size; + struct wmi_link_stats_record *hdr; + + if (payload_size < hdr_size) { + wil_err(wil, "link stats wrong event size %zu\n", payload_size); + return; + } + + while (payload_size >= hdr_size) { + hdr = payload; + stats_size = le16_to_cpu(hdr->record_size); + record_size = hdr_size + stats_size; + + if (payload_size < record_size) { + wil_err(wil, "link stats payload ended unexpectedly, size %zu < %zu\n", + payload_size, record_size); + return; + } + + switch (hdr->record_type_id) { + case WMI_LINK_STATS_TYPE_BASIC: + expected_size = sizeof(struct wmi_link_stats_basic); + if (stats_size < expected_size) { + wil_err(wil, "link stats invalid basic record size %zu < %zu\n", + stats_size, expected_size); + return; + } + if (vif->fw_stats_ready) { + /* clean old statistics */ + vif->fw_stats_tsf = 0; + vif->fw_stats_ready = 0; + } + + wil_link_stats_store_basic(vif, payload + hdr_size); + + if (!has_next) { + vif->fw_stats_tsf = tsf; + vif->fw_stats_ready = 1; + } + + break; + case WMI_LINK_STATS_TYPE_GLOBAL: + expected_size = sizeof(struct wmi_link_stats_global); + if (stats_size < sizeof(struct wmi_link_stats_global)) { + wil_err(wil, "link stats invalid global record size %zu < %zu\n", + stats_size, expected_size); + return; + } + + if (wil->fw_stats_global.ready) { + /* clean old statistics */ + wil->fw_stats_global.tsf = 0; + wil->fw_stats_global.ready = 0; + } + + wil_link_stats_store_global(vif, payload + hdr_size); + + if (!has_next) { + wil->fw_stats_global.tsf = tsf; + wil->fw_stats_global.ready = 1; + } + + break; + default: + break; + } + + /* skip to next record */ + payload += record_size; + payload_size -= record_size; + } +} + +static void +wmi_evt_link_stats(struct wil6210_vif *vif, int id, void *d, int len) +{ + struct wil6210_priv *wil = vif_to_wil(vif); + struct wmi_link_stats_event *evt = d; + size_t payload_size; + + if (len < offsetof(struct wmi_link_stats_event, payload)) { + wil_err(wil, "stats event way too short %d\n", len); + return; + } + payload_size = le16_to_cpu(evt->payload_size); + if (len < sizeof(struct wmi_link_stats_event) + payload_size) { + wil_err(wil, "stats event too short %d\n", len); + return; + } + + wmi_link_stats_parse(vif, le64_to_cpu(evt->tsf), evt->has_next, + evt->payload, payload_size); +} + /** * Some events are ignored for purpose; and need not be interpreted as * "unhandled events" @@ -1359,6 +1491,7 @@ static const struct { {WMI_RING_EN_EVENTID, wmi_evt_ring_en}, {WMI_DATA_PORT_OPEN_EVENTID, wmi_evt_ignore}, {WMI_SCHED_SCAN_RESULT_EVENTID, wmi_evt_sched_scan_result}, + {WMI_LINK_STATS_EVENTID, wmi_evt_link_stats}, }; /* @@ -2983,6 +3116,60 @@ int wmi_mgmt_tx(struct wil6210_vif *vif, const u8 *buf, size_t len) return rc; } +int wmi_mgmt_tx_ext(struct wil6210_vif *vif, const u8 *buf, size_t len, + u8 channel, u16 duration_ms) +{ + size_t total; + struct wil6210_priv *wil = vif_to_wil(vif); + struct ieee80211_mgmt *mgmt_frame = (void *)buf; + struct wmi_sw_tx_req_ext_cmd *cmd; + struct { + struct wmi_cmd_hdr wmi; + struct wmi_sw_tx_complete_event evt; + } __packed evt = { + .evt = {.status = WMI_FW_STATUS_FAILURE}, + }; + int rc; + + wil_dbg_wmi(wil, "mgmt_tx_ext mid %d channel %d duration %d\n", + vif->mid, channel, duration_ms); + wil_hex_dump_wmi("mgmt_tx_ext frame ", DUMP_PREFIX_OFFSET, 16, 1, buf, + len, true); + + if (len < sizeof(struct ieee80211_hdr_3addr)) { + wil_err(wil, "short frame. len %zu\n", len); + return -EINVAL; + } + + total = sizeof(*cmd) + len; + if (total < len) { + wil_err(wil, "mgmt_tx_ext invalid len %zu\n", len); + return -EINVAL; + } + + cmd = kzalloc(total, GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + memcpy(cmd->dst_mac, mgmt_frame->da, WMI_MAC_LEN); + cmd->len = cpu_to_le16(len); + memcpy(cmd->payload, buf, len); + cmd->channel = channel - 1; + cmd->duration_ms = cpu_to_le16(duration_ms); + + rc = wmi_call(wil, WMI_SW_TX_REQ_EXT_CMDID, vif->mid, cmd, total, + WMI_SW_TX_COMPLETE_EVENTID, &evt, sizeof(evt), 2000); + if (!rc && evt.evt.status != WMI_FW_STATUS_SUCCESS) { + wil_err(wil, "mgmt_tx_ext failed with status %d\n", + evt.evt.status); + rc = -EINVAL; + } + + kfree(cmd); + + return rc; +} + int wil_wmi_tx_sring_cfg(struct wil6210_priv *wil, int ring_id) { int rc; @@ -3242,3 +3429,37 @@ int wil_wmi_bcast_desc_ring_add(struct wil6210_vif *vif, int ring_id) return 0; } + +int wmi_link_stats_cfg(struct wil6210_vif *vif, u32 type, u8 cid, u32 interval) +{ + struct wil6210_priv *wil = vif_to_wil(vif); + struct wmi_link_stats_cmd cmd = { + .record_type_mask = cpu_to_le32(type), + .cid = cid, + .action = WMI_LINK_STATS_SNAPSHOT, + .interval_msec = cpu_to_le32(interval), + }; + struct { + struct wmi_cmd_hdr wmi; + struct wmi_link_stats_config_done_event evt; + } __packed reply = { + .evt = {.status = WMI_FW_STATUS_FAILURE}, + }; + int rc; + + rc = wmi_call(wil, WMI_LINK_STATS_CMDID, vif->mid, &cmd, sizeof(cmd), + WMI_LINK_STATS_CONFIG_DONE_EVENTID, &reply, + sizeof(reply), WIL_WMI_CALL_GENERAL_TO_MS); + if (rc) { + wil_err(wil, "WMI_LINK_STATS_CMDID failed, rc %d\n", rc); + return rc; + } + + if (reply.evt.status != WMI_FW_STATUS_SUCCESS) { + wil_err(wil, "Link statistics config failed, status %d\n", + reply.evt.status); + return -EINVAL; + } + + return 0; +} diff --git a/drivers/net/wireless/ath/wil6210/wmi.h b/drivers/net/wireless/ath/wil6210/wmi.h index abf6f05c4801..139acb2caf92 100644 --- a/drivers/net/wireless/ath/wil6210/wmi.h +++ b/drivers/net/wireless/ath/wil6210/wmi.h @@ -53,6 +53,17 @@ * must always be kept equal to (WMI_RF_RX2TX_LENGTH+1) */ #define WMI_RF_RX2TX_CONF_LENGTH (4) +/* Qos configuration */ +#define WMI_QOS_NUM_OF_PRIORITY (4) +#define WMI_QOS_MIN_DEFAULT_WEIGHT (10) +#define WMI_QOS_VRING_SLOT_MIN_MS (2) +#define WMI_QOS_VRING_SLOT_MAX_MS (10) +/* (WMI_QOS_MIN_DEFAULT_WEIGHT * WMI_QOS_VRING_SLOT_MAX_MS / + * WMI_QOS_VRING_SLOT_MIN_MS) + */ +#define WMI_QOS_MAX_WEIGHT 50 +#define WMI_QOS_SET_VIF_PRIORITY (0xFF) +#define WMI_QOS_DEFAULT_PRIORITY (WMI_QOS_NUM_OF_PRIORITY) /* Mailbox interface * used for commands and events @@ -86,7 +97,12 @@ enum wmi_fw_capability { WMI_FW_CAPABILITY_PNO = 15, WMI_FW_CAPABILITY_REF_CLOCK_CONTROL = 18, WMI_FW_CAPABILITY_AP_SME_OFFLOAD_NONE = 19, + WMI_FW_CAPABILITY_MULTI_VIFS = 20, + WMI_FW_CAPABILITY_FT_ROAMING = 21, + WMI_FW_CAPABILITY_BACK_WIN_SIZE_64 = 22, WMI_FW_CAPABILITY_AMSDU = 23, + WMI_FW_CAPABILITY_RAW_MODE = 24, + WMI_FW_CAPABILITY_TX_REQ_EXT = 25, WMI_FW_CAPABILITY_MAX, }; @@ -110,6 +126,9 @@ enum wmi_command_id { WMI_SET_PROBED_SSID_CMDID = 0x0A, /* deprecated */ WMI_SET_LISTEN_INT_CMDID = 0x0B, + WMI_FT_AUTH_CMDID = 0x0C, + WMI_FT_REASSOC_CMDID = 0x0D, + WMI_UPDATE_FT_IES_CMDID = 0x0E, WMI_BCON_CTRL_CMDID = 0x0F, WMI_ADD_CIPHER_KEY_CMDID = 0x16, WMI_DELETE_CIPHER_KEY_CMDID = 0x17, @@ -118,6 +137,12 @@ enum wmi_command_id { WMI_SET_WSC_STATUS_CMDID = 0x41, WMI_PXMT_RANGE_CFG_CMDID = 0x42, WMI_PXMT_SNR2_RANGE_CFG_CMDID = 0x43, + WMI_RADAR_GENERAL_CONFIG_CMDID = 0x100, + WMI_RADAR_CONFIG_SELECT_CMDID = 0x101, + WMI_RADAR_PARAMS_CONFIG_CMDID = 0x102, + WMI_RADAR_SET_MODE_CMDID = 0x103, + WMI_RADAR_CONTROL_CMDID = 0x104, + WMI_RADAR_PCI_CONTROL_CMDID = 0x105, WMI_MEM_READ_CMDID = 0x800, WMI_MEM_WR_CMDID = 0x801, WMI_ECHO_CMDID = 0x803, @@ -158,6 +183,10 @@ enum wmi_command_id { WMI_SET_PCP_CHANNEL_CMDID = 0x829, WMI_GET_PCP_CHANNEL_CMDID = 0x82A, WMI_SW_TX_REQ_CMDID = 0x82B, + /* Event is shared between WMI_SW_TX_REQ_CMDID and + * WMI_SW_TX_REQ_EXT_CMDID + */ + WMI_SW_TX_REQ_EXT_CMDID = 0x82C, WMI_MLME_PUSH_CMDID = 0x835, WMI_BEAMFORMING_MGMT_CMDID = 0x836, WMI_BF_TXSS_MGMT_CMDID = 0x837, @@ -207,7 +236,12 @@ enum wmi_command_id { WMI_GET_PCP_FACTOR_CMDID = 0x91B, /* Power Save Configuration Commands */ WMI_PS_DEV_PROFILE_CFG_CMDID = 0x91C, + WMI_RS_ENABLE_CMDID = 0x91E, + WMI_RS_CFG_EX_CMDID = 0x91F, + WMI_GET_DETAILED_RS_RES_EX_CMDID = 0x920, + /* deprecated */ WMI_RS_CFG_CMDID = 0x921, + /* deprecated */ WMI_GET_DETAILED_RS_RES_CMDID = 0x922, WMI_AOA_MEAS_CMDID = 0x923, WMI_BRP_SET_ANT_LIMIT_CMDID = 0x924, @@ -236,7 +270,9 @@ enum wmi_command_id { WMI_PRIO_TX_SECTORS_ORDER_CMDID = 0x9A5, WMI_PRIO_TX_SECTORS_NUMBER_CMDID = 0x9A6, WMI_PRIO_TX_SECTORS_SET_DEFAULT_CFG_CMDID = 0x9A7, + /* deprecated */ WMI_BF_CONTROL_CMDID = 0x9AA, + WMI_BF_CONTROL_EX_CMDID = 0x9AB, WMI_TX_STATUS_RING_ADD_CMDID = 0x9C0, WMI_RX_STATUS_RING_ADD_CMDID = 0x9C1, WMI_TX_DESC_RING_ADD_CMDID = 0x9C2, @@ -252,6 +288,11 @@ enum wmi_command_id { WMI_GET_CCA_INDICATIONS_CMDID = 0xA07, WMI_SET_CCA_INDICATIONS_BI_AVG_NUM_CMDID = 0xA08, WMI_INTERNAL_FW_IOCTL_CMDID = 0xA0B, + WMI_LINK_STATS_CMDID = 0xA0C, + WMI_SET_GRANT_MCS_CMDID = 0xA0E, + WMI_SET_AP_SLOT_SIZE_CMDID = 0xA0F, + WMI_SET_VRING_PRIORITY_WEIGHT_CMDID = 0xA10, + WMI_SET_VRING_PRIORITY_CMDID = 0xA11, WMI_SET_MAC_ADDRESS_CMDID = 0xF003, WMI_ABORT_SCAN_CMDID = 0xF007, WMI_SET_PROMISCUOUS_MODE_CMDID = 0xF041, @@ -450,6 +491,30 @@ struct wmi_start_sched_scan_cmd { struct wmi_sched_scan_plan scan_plans[WMI_MAX_PLANS_NUM]; } __packed; +/* WMI_FT_AUTH_CMDID */ +struct wmi_ft_auth_cmd { + u8 bssid[WMI_MAC_LEN]; + /* enum wmi_channel */ + u8 channel; + /* enum wmi_channel */ + u8 edmg_channel; + u8 reserved[4]; +} __packed; + +/* WMI_FT_REASSOC_CMDID */ +struct wmi_ft_reassoc_cmd { + u8 bssid[WMI_MAC_LEN]; + u8 reserved[2]; +} __packed; + +/* WMI_UPDATE_FT_IES_CMDID */ +struct wmi_update_ft_ies_cmd { + /* Length of the FT IEs */ + __le16 ie_len; + u8 reserved[2]; + u8 ie_info[0]; +} __packed; + /* WMI_SET_PROBED_SSID_CMDID */ #define MAX_PROBED_SSID_INDEX (3) @@ -506,6 +571,109 @@ struct wmi_pxmt_snr2_range_cfg_cmd { s8 snr2range_arr[2]; } __packed; +/* WMI_RADAR_GENERAL_CONFIG_CMDID */ +struct wmi_radar_general_config_cmd { + /* Number of pulses (CIRs) in FW FIFO to initiate pulses transfer + * from FW to Host + */ + __le32 fifo_watermark; + /* In unit of us, in the range [100, 1000000] */ + __le32 t_burst; + /* Valid in the range [1, 32768], 0xFFFF means infinite */ + __le32 n_bursts; + /* In unit of 330Mhz clk, in the range [4, 2000]*330 */ + __le32 t_pulse; + /* In the range of [1,4096] */ + __le16 n_pulses; + /* Number of taps after cTap per CIR */ + __le16 n_samples; + /* Offset from the main tap (0 = zero-distance). In the range of [0, + * 255] + */ + u8 first_sample_offset; + /* Number of Pulses to average, 1, 2, 4, 8 */ + u8 pulses_to_avg; + /* Number of adjacent taps to average, 1, 2, 4, 8 */ + u8 samples_to_avg; + /* The index to config general params */ + u8 general_index; + u8 reserved[4]; +} __packed; + +/* WMI_RADAR_CONFIG_SELECT_CMDID */ +struct wmi_radar_config_select_cmd { + /* Select the general params index to use */ + u8 general_index; + u8 reserved[3]; + /* 0 means don't update burst_active_vector */ + __le32 burst_active_vector; + /* 0 means don't update pulse_active_vector */ + __le32 pulse_active_vector; +} __packed; + +/* WMI_RADAR_PARAMS_CONFIG_CMDID */ +struct wmi_radar_params_config_cmd { + /* The burst index selected to config */ + u8 burst_index; + /* 0-not active, 1-active */ + u8 burst_en; + /* The pulse index selected to config */ + u8 pulse_index; + /* 0-not active, 1-active */ + u8 pulse_en; + /* TX RF to use on current pulse */ + u8 tx_rfc_idx; + u8 tx_sector; + /* Offset from calibrated value.(expected to be 0)(value is row in + * Gain-LUT, not dB) + */ + s8 tx_rf_gain_comp; + /* expected to be 0 */ + s8 tx_bb_gain_comp; + /* RX RF to use on current pulse */ + u8 rx_rfc_idx; + u8 rx_sector; + /* Offset from calibrated value.(expected to be 0)(value is row in + * Gain-LUT, not dB) + */ + s8 rx_rf_gain_comp; + /* Value in dB.(expected to be 0) */ + s8 rx_bb_gain_comp; + /* Offset from calibrated value.(expected to be 0) */ + s8 rx_timing_offset; + u8 reserved[3]; +} __packed; + +/* WMI_RADAR_SET_MODE_CMDID */ +struct wmi_radar_set_mode_cmd { + /* 0-disable/1-enable */ + u8 enable; + /* enum wmi_channel */ + u8 channel; + /* In the range of [0,7], 0xff means use default */ + u8 tx_rfc_idx; + /* In the range of [0,7], 0xff means use default */ + u8 rx_rfc_idx; +} __packed; + +/* WMI_RADAR_CONTROL_CMDID */ +struct wmi_radar_control_cmd { + /* 0-stop/1-start */ + u8 start; + u8 reserved[3]; +} __packed; + +/* WMI_RADAR_PCI_CONTROL_CMDID */ +struct wmi_radar_pci_control_cmd { + /* pcie host buffer start address */ + __le64 base_addr; + /* pcie host control block address */ + __le64 control_block_addr; + /* pcie host buffer size */ + __le32 buffer_size; + __le32 reserved; +} __packed; + /* WMI_RF_MGMT_CMDID */ enum wmi_rf_mgmt_type { WMI_RF_MGMT_W_DISABLE = 0x00, @@ -643,12 +811,18 @@ struct wmi_pcp_start_cmd { u8 pcp_max_assoc_sta; u8 hidden_ssid; u8 is_go; - u8 reserved0[5]; + /* enum wmi_channel WMI_CHANNEL_9..WMI_CHANNEL_12 */ + u8 edmg_channel; + u8 raw_mode; + u8 reserved[3]; /* A-BFT length override if non-0 */ u8 abft_len; /* enum wmi_ap_sme_offload_mode_e */ u8 ap_sme_offload_mode; u8 network_type; + /* enum wmi_channel WMI_CHANNEL_1..WMI_CHANNEL_6; for EDMG this is + * the primary channel number + */ u8 channel; u8 disable_sec_offload; u8 disable_sec; @@ -661,6 +835,17 @@ struct wmi_sw_tx_req_cmd { u8 payload[0]; } __packed; +/* WMI_SW_TX_REQ_EXT_CMDID */ +struct wmi_sw_tx_req_ext_cmd { + u8 dst_mac[WMI_MAC_LEN]; + __le16 len; + __le16 duration_ms; + /* Channel to use, 0xFF for currently active channel */ + u8 channel; + u8 reserved[5]; + u8 payload[0]; +} __packed; + /* WMI_VRING_SWITCH_TIMING_CONFIG_CMDID */ struct wmi_vring_switch_timing_config_cmd { /* Set vring timing configuration: @@ -687,6 +872,7 @@ struct wmi_vring_cfg_schd { enum wmi_vring_cfg_encap_trans_type { WMI_VRING_ENC_TYPE_802_3 = 0x00, WMI_VRING_ENC_TYPE_NATIVE_WIFI = 0x01, + WMI_VRING_ENC_TYPE_NONE = 0x02, }; enum wmi_vring_cfg_ds_cfg { @@ -744,7 +930,11 @@ struct wmi_vring_cfg { u8 cid; /* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */ u8 tid; - u8 reserved[2]; + /* Update the vring's priority for Qos purpose. Set to + * WMI_QOS_DEFAULT_PRIORITY to use MID's QoS priority + */ + u8 qos_priority; + u8 reserved; } __packed; enum wmi_vring_cfg_cmd_action { @@ -775,20 +965,6 @@ struct wmi_bcast_vring_cfg_cmd { struct wmi_bcast_vring_cfg vring_cfg; } __packed; -/* WMI_LO_POWER_CALIB_FROM_OTP_CMDID */ -struct wmi_lo_power_calib_from_otp_cmd { - /* index to read from OTP. zero based */ - u8 index; - u8 reserved[3]; -} __packed; - -/* WMI_LO_POWER_CALIB_FROM_OTP_EVENTID */ -struct wmi_lo_power_calib_from_otp_event { - /* wmi_fw_status */ - u8 status; - u8 reserved[3]; -} __packed; - struct wmi_edma_ring_cfg { __le64 ring_mem_base; /* size in number of items */ @@ -861,6 +1037,20 @@ struct wmi_bcast_desc_ring_add_cmd { u8 reserved[4]; } __packed; +/* WMI_LO_POWER_CALIB_FROM_OTP_CMDID */ +struct wmi_lo_power_calib_from_otp_cmd { + /* index to read from OTP. zero based */ + u8 index; + u8 reserved[3]; +} __packed; + +/* WMI_LO_POWER_CALIB_FROM_OTP_EVENTID */ +struct wmi_lo_power_calib_from_otp_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + /* WMI_RING_BA_EN_CMDID */ struct wmi_ring_ba_en_cmd { u8 ring_id; @@ -1094,8 +1284,8 @@ struct wmi_echo_cmd { } __packed; /* WMI_DEEP_ECHO_CMDID - * Check FW and ucode are alive - * Returned event: WMI_ECHO_RSP_EVENTID + * Check FW and uCode is alive + * Returned event: WMI_DEEP_ECHO_RSP_EVENTID */ struct wmi_deep_echo_cmd { __le32 value; @@ -1419,6 +1609,10 @@ struct wmi_fixed_scheduling_config_complete_event { u8 reserved[3]; } __packed; +/* This value exists for backwards compatibility only. + * Do not use it in new commands. + * Use dynamic arrays where possible. + */ #define WMI_NUM_MCS (13) /* WMI_FIXED_SCHEDULING_CONFIG_CMDID */ @@ -1466,6 +1660,52 @@ struct wmi_set_multi_directed_omnis_config_event { u8 reserved[3]; } __packed; +/* WMI_RADAR_GENERAL_CONFIG_EVENTID */ +struct wmi_radar_general_config_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_RADAR_CONFIG_SELECT_EVENTID */ +struct wmi_radar_config_select_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; + /* In unit of bytes */ + __le32 fifo_size; + /* In unit of bytes */ + __le32 pulse_size; +} __packed; + +/* WMI_RADAR_PARAMS_CONFIG_EVENTID */ +struct wmi_radar_params_config_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_RADAR_SET_MODE_EVENTID */ +struct wmi_radar_set_mode_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_RADAR_CONTROL_EVENTID */ +struct wmi_radar_control_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_RADAR_PCI_CONTROL_EVENTID */ +struct wmi_radar_pci_control_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + /* WMI_SET_LONG_RANGE_CONFIG_CMDID */ struct wmi_set_long_range_config_cmd { __le32 reserved; @@ -1478,12 +1718,12 @@ struct wmi_set_long_range_config_complete_event { u8 reserved[3]; } __packed; -/* payload max size is 236 bytes: max event buffer size (256) - WMI headers +/* payload max size is 1024 bytes: max event buffer size (1044) - WMI headers * (16) - prev struct field size (4) */ -#define WMI_MAX_IOCTL_PAYLOAD_SIZE (236) -#define WMI_MAX_IOCTL_REPLY_PAYLOAD_SIZE (236) -#define WMI_MAX_INTERNAL_EVENT_PAYLOAD_SIZE (236) +#define WMI_MAX_IOCTL_PAYLOAD_SIZE (1024) +#define WMI_MAX_IOCTL_REPLY_PAYLOAD_SIZE (1024) +#define WMI_MAX_INTERNAL_EVENT_PAYLOAD_SIZE (1024) enum wmi_internal_fw_ioctl_code { WMI_INTERNAL_FW_CODE_NONE = 0x0, @@ -1523,7 +1763,37 @@ struct wmi_internal_fw_event_event { __le32 payload[0]; } __packed; -/* WMI_BF_CONTROL_CMDID */ +/* WMI_SET_VRING_PRIORITY_WEIGHT_CMDID */ +struct wmi_set_vring_priority_weight_cmd { + /* Array of weights. Valid values are + * WMI_QOS_MIN_DEFAULT_WEIGHT...WMI_QOS_MAX_WEIGHT. Weight #0 is + * hard-coded WMI_QOS_MIN_WEIGHT. This array provide the weights + * #1..#3 + */ + u8 weight[3]; + u8 reserved; +} __packed; + +/* WMI_SET_VRING_PRIORITY_CMDID */ +struct wmi_vring_priority { + u8 vring_idx; + /* Weight index. Valid value is 0-3 */ + u8 priority; + u8 reserved[2]; +} __packed; + +/* WMI_SET_VRING_PRIORITY_CMDID */ +struct wmi_set_vring_priority_cmd { + /* number of entries in vring_priority. Set to + * WMI_QOS_SET_VIF_PRIORITY to update the VIF's priority, and there + * will be only one entry in vring_priority + */ + u8 num_of_vrings; + u8 reserved[3]; + struct wmi_vring_priority vring_priority[0]; +} __packed; + +/* WMI_BF_CONTROL_CMDID - deprecated */ struct wmi_bf_control_cmd { /* wmi_bf_triggers */ __le32 triggers; @@ -1565,6 +1835,95 @@ struct wmi_bf_control_cmd { u8 reserved2[2]; } __packed; +/* BF configuration for each MCS */ +struct wmi_bf_control_ex_mcs { + /* Long term throughput threshold [Mbps] */ + u8 long_term_mbps_th_tbl; + u8 reserved; + /* Long term timeout threshold table [msec] */ + __le16 long_term_trig_timeout_per_mcs; +} __packed; + +/* WMI_BF_CONTROL_EX_CMDID */ +struct wmi_bf_control_ex_cmd { + /* wmi_bf_triggers */ + __le32 triggers; + /* enum wmi_edmg_tx_mode */ + u8 tx_mode; + /* DISABLED = 0, ENABLED = 1 , DRY_RUN = 2 */ + u8 txss_mode; + /* DISABLED = 0, ENABLED = 1, DRY_RUN = 2 */ + u8 brp_mode; + /* Max cts threshold (correspond to + * WMI_BF_TRIGGER_MAX_CTS_FAILURE_IN_TXOP) + */ + u8 bf_trigger_max_cts_failure_thr; + /* Max cts threshold in dense (correspond to + * WMI_BF_TRIGGER_MAX_CTS_FAILURE_IN_TXOP) + */ + u8 bf_trigger_max_cts_failure_dense_thr; + /* Max b-ack threshold (correspond to + * WMI_BF_TRIGGER_MAX_BACK_FAILURE) + */ + u8 bf_trigger_max_back_failure_thr; + /* Max b-ack threshold in dense (correspond to + * WMI_BF_TRIGGER_MAX_BACK_FAILURE) + */ + u8 bf_trigger_max_back_failure_dense_thr; + u8 reserved0; + /* Wrong sectors threshold */ + __le32 wrong_sector_bis_thr; + /* BOOL to enable/disable long term trigger */ + u8 long_term_enable; + /* 1 = Update long term thresholds from the long_term_mbps_th_tbl and + * long_term_trig_timeout_per_mcs arrays, 0 = Ignore + */ + u8 long_term_update_thr; + u8 each_mcs_cfg_size; + u8 reserved1; + /* Configuration for each MCS */ + struct wmi_bf_control_ex_mcs each_mcs_cfg[0]; +} __packed; + +/* WMI_LINK_STATS_CMD */ +enum wmi_link_stats_action { + WMI_LINK_STATS_SNAPSHOT = 0x00, + WMI_LINK_STATS_PERIODIC = 0x01, + WMI_LINK_STATS_STOP_PERIODIC = 0x02, +}; + +/* WMI_LINK_STATS_EVENT record identifiers */ +enum wmi_link_stats_record_type { + WMI_LINK_STATS_TYPE_BASIC = 0x01, + WMI_LINK_STATS_TYPE_GLOBAL = 0x02, +}; + +/* WMI_LINK_STATS_CMDID */ +struct wmi_link_stats_cmd { + /* bitmask of required record types + * (wmi_link_stats_record_type_e) + */ + __le32 record_type_mask; + /* 0xff for all cids */ + u8 cid; + /* wmi_link_stats_action_e */ + u8 action; + u8 reserved[6]; + /* range = 100 - 10000 */ + __le32 interval_msec; +} __packed; + +/* WMI_SET_GRANT_MCS_CMDID */ +struct wmi_set_grant_mcs_cmd { + u8 mcs; + u8 reserved[3]; +} __packed; + +/* WMI_SET_AP_SLOT_SIZE_CMDID */ +struct wmi_set_ap_slot_size_cmd { + __le32 slot_size; +} __packed; + /* WMI Events * List of Events (target to host) */ @@ -1577,10 +1936,19 @@ enum wmi_event_id { WMI_SCHED_SCAN_RESULT_EVENTID = 0x1007, WMI_SCAN_COMPLETE_EVENTID = 0x100A, WMI_REPORT_STATISTICS_EVENTID = 0x100B, + WMI_FT_AUTH_STATUS_EVENTID = 0x100C, + WMI_FT_REASSOC_STATUS_EVENTID = 0x100D, + WMI_RADAR_GENERAL_CONFIG_EVENTID = 0x1100, + WMI_RADAR_CONFIG_SELECT_EVENTID = 0x1101, + WMI_RADAR_PARAMS_CONFIG_EVENTID = 0x1102, + WMI_RADAR_SET_MODE_EVENTID = 0x1103, + WMI_RADAR_CONTROL_EVENTID = 0x1104, + WMI_RADAR_PCI_CONTROL_EVENTID = 0x1105, WMI_RD_MEM_RSP_EVENTID = 0x1800, WMI_FW_READY_EVENTID = 0x1801, WMI_EXIT_FAST_MEM_ACC_MODE_EVENTID = 0x200, WMI_ECHO_RSP_EVENTID = 0x1803, + WMI_DEEP_ECHO_RSP_EVENTID = 0x1804, /* deprecated */ WMI_FS_TUNE_DONE_EVENTID = 0x180A, /* deprecated */ @@ -1606,6 +1974,9 @@ enum wmi_event_id { WMI_DELBA_EVENTID = 0x1826, WMI_GET_SSID_EVENTID = 0x1828, WMI_GET_PCP_CHANNEL_EVENTID = 0x182A, + /* Event is shared between WMI_SW_TX_REQ_CMDID and + * WMI_SW_TX_REQ_EXT_CMDID + */ WMI_SW_TX_COMPLETE_EVENTID = 0x182B, WMI_BEAMFORMING_MGMT_DONE_EVENTID = 0x1836, WMI_BF_TXSS_MGMT_DONE_EVENTID = 0x1837, @@ -1653,7 +2024,12 @@ enum wmi_event_id { WMI_PCP_FACTOR_EVENTID = 0x191A, /* Power Save Configuration Events */ WMI_PS_DEV_PROFILE_CFG_EVENTID = 0x191C, + WMI_RS_ENABLE_EVENTID = 0x191E, + WMI_RS_CFG_EX_EVENTID = 0x191F, + WMI_GET_DETAILED_RS_RES_EX_EVENTID = 0x1920, + /* deprecated */ WMI_RS_CFG_DONE_EVENTID = 0x1921, + /* deprecated */ WMI_GET_DETAILED_RS_RES_EVENTID = 0x1922, WMI_AOA_MEAS_EVENTID = 0x1923, WMI_BRP_SET_ANT_LIMIT_EVENTID = 0x1924, @@ -1681,7 +2057,9 @@ enum wmi_event_id { WMI_PRIO_TX_SECTORS_ORDER_EVENTID = 0x19A5, WMI_PRIO_TX_SECTORS_NUMBER_EVENTID = 0x19A6, WMI_PRIO_TX_SECTORS_SET_DEFAULT_CFG_EVENTID = 0x19A7, + /* deprecated */ WMI_BF_CONTROL_EVENTID = 0x19AA, + WMI_BF_CONTROL_EX_EVENTID = 0x19AB, WMI_TX_STATUS_RING_CFG_DONE_EVENTID = 0x19C0, WMI_RX_STATUS_RING_CFG_DONE_EVENTID = 0x19C1, WMI_TX_DESC_RING_CFG_DONE_EVENTID = 0x19C2, @@ -1697,6 +2075,12 @@ enum wmi_event_id { WMI_SET_CCA_INDICATIONS_BI_AVG_NUM_EVENTID = 0x1A08, WMI_INTERNAL_FW_EVENT_EVENTID = 0x1A0A, WMI_INTERNAL_FW_IOCTL_EVENTID = 0x1A0B, + WMI_LINK_STATS_CONFIG_DONE_EVENTID = 0x1A0C, + WMI_LINK_STATS_EVENTID = 0x1A0D, + WMI_SET_GRANT_MCS_EVENTID = 0x1A0E, + WMI_SET_AP_SLOT_SIZE_EVENTID = 0x1A0F, + WMI_SET_VRING_PRIORITY_WEIGHT_EVENTID = 0x1A10, + WMI_SET_VRING_PRIORITY_EVENTID = 0x1A11, WMI_SET_CHANNEL_EVENTID = 0x9000, WMI_ASSOC_REQ_EVENTID = 0x9001, WMI_EAPOL_RX_EVENTID = 0x9002, @@ -1961,6 +2345,33 @@ struct wmi_scan_complete_event { __le32 status; } __packed; +/* WMI_FT_AUTH_STATUS_EVENTID */ +struct wmi_ft_auth_status_event { + /* enum wmi_fw_status */ + u8 status; + u8 reserved[3]; + u8 mac_addr[WMI_MAC_LEN]; + __le16 ie_len; + u8 ie_info[0]; +} __packed; + +/* WMI_FT_REASSOC_STATUS_EVENTID */ +struct wmi_ft_reassoc_status_event { + /* enum wmi_fw_status */ + u8 status; + /* association id received from new AP */ + u8 aid; + /* enum wmi_channel */ + u8 channel; + /* enum wmi_channel */ + u8 edmg_channel; + u8 mac_addr[WMI_MAC_LEN]; + __le16 beacon_ie_len; + __le16 reassoc_req_ie_len; + __le16 reassoc_resp_ie_len; + u8 ie_info[0]; +} __packed; + /* wmi_rx_mgmt_info */ struct wmi_rx_mgmt_info { u8 mcs; @@ -2317,6 +2728,11 @@ struct wmi_echo_rsp_event { __le32 echoed_value; } __packed; +/* WMI_DEEP_ECHO_RSP_EVENTID */ +struct wmi_deep_echo_rsp_event { + __le32 echoed_value; +} __packed; + /* WMI_RF_PWR_ON_DELAY_RSP_EVENTID */ struct wmi_rf_pwr_on_delay_rsp_event { /* wmi_fw_status */ @@ -2455,6 +2871,81 @@ struct wmi_rs_cfg { __le32 mcs_en_vec; } __packed; +enum wmi_edmg_tx_mode { + WMI_TX_MODE_DMG = 0x0, + WMI_TX_MODE_EDMG_CB1 = 0x1, + WMI_TX_MODE_EDMG_CB2 = 0x2, + WMI_TX_MODE_EDMG_CB1_LONG_LDPC = 0x3, + WMI_TX_MODE_EDMG_CB2_LONG_LDPC = 0x4, + WMI_TX_MODE_MAX, +}; + +/* Rate search parameters common configuration */ +struct wmi_rs_cfg_ex_common { + /* enum wmi_edmg_tx_mode */ + u8 mode; + /* stop threshold [0-100] */ + u8 stop_th; + /* MCS1 stop threshold [0-100] */ + u8 mcs1_fail_th; + u8 max_back_failure_th; + /* Debug feature for disabling internal RS trigger (which is + * currently triggered by BF Done) + */ + u8 dbg_disable_internal_trigger; + u8 reserved[3]; + __le32 back_failure_mask; +} __packed; + +/* Rate search parameters configuration per MCS */ +struct wmi_rs_cfg_ex_mcs { + /* The maximal allowed PER for each MCS + * MCS will be considered as failed if PER during RS is higher + */ + u8 per_threshold; + /* Number of MPDUs for each MCS + * this is the minimal statistic required to make an educated + * decision + */ + u8 min_frame_cnt; + u8 reserved[2]; +} __packed; + +/* WMI_RS_CFG_EX_CMDID */ +struct wmi_rs_cfg_ex_cmd { + /* Configuration for all MCSs */ + struct wmi_rs_cfg_ex_common common_cfg; + u8 each_mcs_cfg_size; + u8 reserved[3]; + /* Configuration for each MCS */ + struct wmi_rs_cfg_ex_mcs each_mcs_cfg[0]; +} __packed; + +/* WMI_RS_CFG_EX_EVENTID */ +struct wmi_rs_cfg_ex_event { + /* enum wmi_edmg_tx_mode */ + u8 mode; + /* enum wmi_fw_status */ + u8 status; + u8 reserved[2]; +} __packed; + +/* WMI_RS_ENABLE_CMDID */ +struct wmi_rs_enable_cmd { + u8 cid; + /* enable or disable rate search */ + u8 rs_enable; + u8 reserved[2]; + __le32 mcs_en_vec; +} __packed; + +/* WMI_RS_ENABLE_EVENTID */ +struct wmi_rs_enable_event { + /* enum wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + /* Slot types */ enum wmi_sched_scheme_slot_type { WMI_SCHED_SLOT_SP = 0x0, @@ -2547,7 +3038,7 @@ struct wmi_scheduling_scheme_event { u8 reserved[1]; } __packed; -/* WMI_RS_CFG_CMDID */ +/* WMI_RS_CFG_CMDID - deprecated */ struct wmi_rs_cfg_cmd { /* connection id */ u8 cid; @@ -2557,7 +3048,7 @@ struct wmi_rs_cfg_cmd { struct wmi_rs_cfg rs_cfg; } __packed; -/* WMI_RS_CFG_DONE_EVENTID */ +/* WMI_RS_CFG_DONE_EVENTID - deprecated */ struct wmi_rs_cfg_done_event { u8 cid; /* enum wmi_fw_status */ @@ -2565,7 +3056,7 @@ struct wmi_rs_cfg_done_event { u8 reserved[2]; } __packed; -/* WMI_GET_DETAILED_RS_RES_CMDID */ +/* WMI_GET_DETAILED_RS_RES_CMDID - deprecated */ struct wmi_get_detailed_rs_res_cmd { /* connection id */ u8 cid; @@ -2590,7 +3081,7 @@ struct wmi_rs_results { u8 mcs; } __packed; -/* WMI_GET_DETAILED_RS_RES_EVENTID */ +/* WMI_GET_DETAILED_RS_RES_EVENTID - deprecated */ struct wmi_get_detailed_rs_res_event { u8 cid; /* enum wmi_rs_results_status */ @@ -2600,6 +3091,45 @@ struct wmi_get_detailed_rs_res_event { u8 reserved[3]; } __packed; +/* WMI_GET_DETAILED_RS_RES_EX_CMDID */ +struct wmi_get_detailed_rs_res_ex_cmd { + u8 cid; + u8 reserved[3]; +} __packed; + +/* Rate search results */ +struct wmi_rs_results_ex_common { + /* RS timestamp */ + __le32 tsf; + /* RS selected MCS */ + u8 mcs; + /* enum wmi_edmg_tx_mode */ + u8 mode; + u8 reserved[2]; +} __packed; + +/* Rate search results */ +struct wmi_rs_results_ex_mcs { + /* number of sent MPDUs */ + u8 num_of_tx_pkt; + /* number of non-acked MPDUs */ + u8 num_of_non_acked_pkt; + u8 reserved[2]; +} __packed; + +/* WMI_GET_DETAILED_RS_RES_EX_EVENTID */ +struct wmi_get_detailed_rs_res_ex_event { + u8 cid; + /* enum wmi_rs_results_status */ + u8 status; + u8 reserved0[2]; + struct wmi_rs_results_ex_common common_rs_results; + u8 each_mcs_results_size; + u8 reserved1[3]; + /* Results for each MCS */ + struct wmi_rs_results_ex_mcs each_mcs_results[0]; +} __packed; + /* BRP antenna limit mode */ enum wmi_brp_ant_limit_mode { /* Disable BRP force antenna limit */ @@ -3350,13 +3880,20 @@ struct wmi_get_assoc_list_res_event { u8 reserved[3]; } __packed; -/* WMI_BF_CONTROL_EVENTID */ +/* WMI_BF_CONTROL_EVENTID - deprecated */ struct wmi_bf_control_event { /* wmi_fw_status */ u8 status; u8 reserved[3]; } __packed; +/* WMI_BF_CONTROL_EX_EVENTID */ +struct wmi_bf_control_ex_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + /* WMI_COMMAND_NOT_SUPPORTED_EVENTID */ struct wmi_command_not_supported_event { /* device id */ @@ -3426,4 +3963,96 @@ struct wmi_internal_fw_set_channel_event { u8 reserved[3]; } __packed; +/* WMI_LINK_STATS_CONFIG_DONE_EVENTID */ +struct wmi_link_stats_config_done_event { + /* wmi_fw_status_e */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_LINK_STATS_EVENTID */ +struct wmi_link_stats_event { + __le64 tsf; + __le16 payload_size; + u8 has_next; + u8 reserved[5]; + /* a stream of wmi_link_stats_record_s */ + u8 payload[0]; +} __packed; + +/* WMI_LINK_STATS_EVENT */ +struct wmi_link_stats_record { + /* wmi_link_stats_record_type_e */ + u8 record_type_id; + u8 reserved; + __le16 record_size; + u8 record[0]; +} __packed; + +/* WMI_LINK_STATS_TYPE_BASIC */ +struct wmi_link_stats_basic { + u8 cid; + s8 rssi; + u8 sqi; + u8 bf_mcs; + u8 per_average; + u8 selected_rfc; + u8 rx_effective_ant_num; + u8 my_rx_sector; + u8 my_tx_sector; + u8 other_rx_sector; + u8 other_tx_sector; + u8 reserved[7]; + /* 1/4 Db units */ + __le16 snr; + __le32 tx_tpt; + __le32 tx_goodput; + __le32 rx_goodput; + __le32 bf_count; + __le32 rx_bcast_frames; +} __packed; + +/* WMI_LINK_STATS_TYPE_GLOBAL */ +struct wmi_link_stats_global { + /* all ack-able frames */ + __le32 rx_frames; + /* all ack-able frames */ + __le32 tx_frames; + __le32 rx_ba_frames; + __le32 tx_ba_frames; + __le32 tx_beacons; + __le32 rx_mic_errors; + __le32 rx_crc_errors; + __le32 tx_fail_no_ack; + u8 reserved[8]; +} __packed; + +/* WMI_SET_GRANT_MCS_EVENTID */ +struct wmi_set_grant_mcs_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_SET_AP_SLOT_SIZE_EVENTID */ +struct wmi_set_ap_slot_size_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_SET_VRING_PRIORITY_WEIGHT_EVENTID */ +struct wmi_set_vring_priority_weight_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + +/* WMI_SET_VRING_PRIORITY_EVENTID */ +struct wmi_set_vring_priority_event { + /* wmi_fw_status */ + u8 status; + u8 reserved[3]; +} __packed; + #endif /* __WILOCITY_WMI_H__ */ diff --git a/drivers/net/wireless/broadcom/b43/leds.c b/drivers/net/wireless/broadcom/b43/leds.c index cb987c2ecc6b..87131f663292 100644 --- a/drivers/net/wireless/broadcom/b43/leds.c +++ b/drivers/net/wireless/broadcom/b43/leds.c @@ -131,7 +131,7 @@ static int b43_register_led(struct b43_wldev *dev, struct b43_led *led, led->wl = dev->wl; led->index = led_index; led->activelow = activelow; - strncpy(led->name, name, sizeof(led->name)); + strlcpy(led->name, name, sizeof(led->name)); atomic_set(&led->state, 0); led->led_dev.name = led->name; diff --git a/drivers/net/wireless/broadcom/b43legacy/leds.c b/drivers/net/wireless/broadcom/b43legacy/leds.c index fd4565389c77..bc922118b6ac 100644 --- a/drivers/net/wireless/broadcom/b43legacy/leds.c +++ b/drivers/net/wireless/broadcom/b43legacy/leds.c @@ -101,7 +101,7 @@ static int b43legacy_register_led(struct b43legacy_wldev *dev, led->dev = dev; led->index = led_index; led->activelow = activelow; - strncpy(led->name, name, sizeof(led->name)); + strlcpy(led->name, name, sizeof(led->name)); led->led_dev.name = led->name; led->led_dev.default_trigger = default_trigger; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 24c4e18e7d80..5444e6213d45 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6926,15 +6926,15 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->d11inf.io_type = (u8)io_type; brcmu_d11_attach(&cfg->d11inf); - err = brcmf_setup_wiphy(wiphy, ifp); - if (err < 0) - goto priv_out; - /* regulatory notifer below needs access to cfg so * assign it now. */ drvr->config = cfg; + err = brcmf_setup_wiphy(wiphy, ifp); + if (err < 0) + goto priv_out; + brcmf_dbg(INFO, "Registering custom regulatory\n"); wiphy->reg_notifier = brcmf_cfg80211_reg_notifier; wiphy->regulatory_flags |= REGULATORY_CUSTOM_REG; diff --git a/drivers/net/wireless/intel/iwlegacy/3945-debug.c b/drivers/net/wireless/intel/iwlegacy/3945-debug.c index c1b4441fb8b2..a2960032be81 100644 --- a/drivers/net/wireless/intel/iwlegacy/3945-debug.c +++ b/drivers/net/wireless/intel/iwlegacy/3945-debug.c @@ -95,7 +95,7 @@ il3945_ucode_rx_stats_read(struct file *file, char __user *user_buf, pos += scnprintf(buf + pos, bufsz - pos, "%-32s current" - "acumulative delta max\n", + "accumulative delta max\n", "Statistics_Rx - OFDM:"); pos += scnprintf(buf + pos, bufsz - pos, diff --git a/drivers/net/wireless/mediatek/mt76/Kconfig b/drivers/net/wireless/mediatek/mt76/Kconfig index 850611ad347a..b6c5f17dca30 100644 --- a/drivers/net/wireless/mediatek/mt76/Kconfig +++ b/drivers/net/wireless/mediatek/mt76/Kconfig @@ -11,6 +11,7 @@ config MT76x2_COMMON config MT76x0U tristate "MediaTek MT76x0U (USB) support" + select MT76_CORE depends on MAC80211 depends on USB help diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c index 1ecd018f12b8..36da1e6bc21a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c @@ -81,15 +81,15 @@ mt76x0_efuse_read(struct mt76x0_dev *dev, u16 addr, u8 *data, return 0; } +#define MT_MAP_READS DIV_ROUND_UP(MT_EFUSE_USAGE_MAP_SIZE, 16) static int mt76x0_efuse_physical_size_check(struct mt76x0_dev *dev) { - const int map_reads = DIV_ROUND_UP(MT_EFUSE_USAGE_MAP_SIZE, 16); - u8 data[map_reads * 16]; + u8 data[MT_MAP_READS * 16]; int ret, i; u32 start = 0, end = 0, cnt_free; - for (i = 0; i < map_reads; i++) { + for (i = 0; i < MT_MAP_READS; i++) { ret = mt76x0_efuse_read(dev, MT_EE_USAGE_MAP_START + i * 16, data + i * 16, MT_EE_PHYSICAL_READ); if (ret) @@ -241,7 +241,7 @@ mt76x0_set_rf_freq_off(struct mt76x0_dev *dev, u8 *eeprom) static void mt76x0_set_lna_gain(struct mt76x0_dev *dev, u8 *eeprom) { - s8 gain; + u8 gain; dev->ee->lna_gain_2ghz = eeprom[MT_EE_LNA_GAIN_2GHZ]; dev->ee->lna_gain_5ghz[0] = eeprom[MT_EE_LNA_GAIN_5GHZ_0]; diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/mac.c b/drivers/net/wireless/mediatek/mt76/mt76x0/mac.c index 95f28492a843..91a84be36d3b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/mac.c @@ -424,8 +424,6 @@ void mt76x0_mac_set_ampdu_factor(struct mt76x0_dev *dev) u8 min_factor = 3; int i; - return; - rcu_read_lock(); for (i = 0; i < ARRAY_SIZE(dev->wcid); i++) { wcid = rcu_dereference(dev->wcid[i]); diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c index df3facc8e5a4..6597f7cb3411 100644 --- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c +++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c @@ -2818,23 +2818,11 @@ static void btc8723b2ant_action_sco(struct btc_coexist *btcoexist) /* sw mechanism */ if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, true, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, true, + false, false); } } @@ -2949,23 +2937,11 @@ static void btc8723b2ant_action_a2dp(struct btc_coexist *btcoexist) /* sw mechanism */ btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw); if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, false, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, false, + false, false); } } @@ -3008,23 +2984,11 @@ static void btc8723b2ant_action_a2dp_pan_hs(struct btc_coexist *btcoexist) /* sw mechanism */ btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw); if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, false, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, false, + false, false); } } @@ -3071,23 +3035,11 @@ static void btc8723b2ant_action_pan_edr(struct btc_coexist *btcoexist) /* sw mechanism */ btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw); if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, false, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, false, + false, false); } } @@ -3121,23 +3073,11 @@ static void btc8723b2ant_action_pan_hs(struct btc_coexist *btcoexist) btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw); if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, false, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, false, + false, false); } } @@ -3189,23 +3129,11 @@ static void btc8723b2ant_action_pan_edr_a2dp(struct btc_coexist *btcoexist) /* sw mechanism */ if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, false, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, false, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, false, + false, false); } } @@ -3264,23 +3192,11 @@ static void btc8723b2ant_action_pan_edr_hid(struct btc_coexist *btcoexist) /* sw mechanism */ if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, true, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, true, + false, false); } } @@ -3336,23 +3252,11 @@ static void btc8723b2ant_action_hid_a2dp_pan_edr(struct btc_coexist *btcoexist) /* sw mechanism */ if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, true, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, true, + false, false); } } @@ -3436,23 +3340,11 @@ static void btc8723b2ant_action_hid_a2dp(struct btc_coexist *btcoexist) /* sw mechanism */ if (BTC_WIFI_BW_HT40 == wifi_bw) { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, true, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, true, true, + false, false); } else { - if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) || - (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } else { - btc8723b2ant_sw_mechanism(btcoexist, false, true, - false, false); - } + btc8723b2ant_sw_mechanism(btcoexist, false, true, + false, false); } } diff --git a/drivers/net/wireless/rsi/rsi_91x_hal.c b/drivers/net/wireless/rsi/rsi_91x_hal.c index 27e6baf12e90..01edf960ff3c 100644 --- a/drivers/net/wireless/rsi/rsi_91x_hal.c +++ b/drivers/net/wireless/rsi/rsi_91x_hal.c @@ -57,7 +57,6 @@ int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb) struct ieee80211_vif *vif; struct rsi_mgmt_desc *mgmt_desc; struct skb_info *tx_params; - struct ieee80211_bss_conf *bss = NULL; struct rsi_xtended_desc *xtend_desc = NULL; u8 header_size; u32 dword_align_bytes = 0; @@ -91,7 +90,6 @@ int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb) tx_params->internal_hdr_size = header_size; memset(&skb->data[0], 0, header_size); - bss = &vif->bss_conf; wh = (struct ieee80211_hdr *)&skb->data[header_size]; mgmt_desc = (struct rsi_mgmt_desc *)skb->data; @@ -148,7 +146,6 @@ int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb) struct ieee80211_hdr *wh = NULL; struct ieee80211_tx_info *info; struct skb_info *tx_params; - struct ieee80211_bss_conf *bss; struct rsi_data_desc *data_desc; struct rsi_xtended_desc *xtend_desc; u8 ieee80211_size = MIN_802_11_HDR_LEN; @@ -159,7 +156,6 @@ int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb) info = IEEE80211_SKB_CB(skb); vif = info->control.vif; - bss = &vif->bss_conf; tx_params = (struct skb_info *)info->driver_data; header_size = FRAME_DESC_SZ + sizeof(struct rsi_xtended_desc); @@ -288,7 +284,6 @@ int rsi_send_data_pkt(struct rsi_common *common, struct sk_buff *skb) struct ieee80211_tx_info *info; struct skb_info *tx_params; struct ieee80211_bss_conf *bss; - struct ieee80211_hdr *wh; int status = -EINVAL; u8 header_size; @@ -304,7 +299,6 @@ int rsi_send_data_pkt(struct rsi_common *common, struct sk_buff *skb) bss = &vif->bss_conf; tx_params = (struct skb_info *)info->driver_data; header_size = tx_params->internal_hdr_size; - wh = (struct ieee80211_hdr *)&skb->data[header_size]; if (((vif->type == NL80211_IFTYPE_STATION) || (vif->type == NL80211_IFTYPE_P2P_CLIENT)) && @@ -747,13 +741,11 @@ static int ping_pong_write(struct rsi_hw *adapter, u8 cmd, u8 *addr, u32 size) static int auto_fw_upgrade(struct rsi_hw *adapter, u8 *flash_content, u32 content_size) { - u8 cmd, *temp_flash_content; + u8 cmd; u32 temp_content_size, num_flash, index; u32 flash_start_address; int status; - temp_flash_content = flash_content; - if (content_size > MAX_FLASH_FILE_SIZE) { rsi_dbg(ERR_ZONE, "%s: Flash Content size is more than 400K %u\n", diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 799cba4624a5..2ff874b6ec89 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -895,7 +895,6 @@ static RING_IDX xennet_fill_frags(struct netfront_queue *queue, struct sk_buff *skb, struct sk_buff_head *list) { - struct skb_shared_info *shinfo = skb_shinfo(skb); RING_IDX cons = queue->rx.rsp_cons; struct sk_buff *nskb; @@ -904,15 +903,16 @@ static RING_IDX xennet_fill_frags(struct netfront_queue *queue, RING_GET_RESPONSE(&queue->rx, ++cons); skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0]; - if (shinfo->nr_frags == MAX_SKB_FRAGS) { + if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) { unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to; BUG_ON(pull_to <= skb_headlen(skb)); __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); } - BUG_ON(shinfo->nr_frags >= MAX_SKB_FRAGS); + BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); - skb_add_rx_frag(skb, shinfo->nr_frags, skb_frag_page(nfrag), + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + skb_frag_page(nfrag), rx->offset, rx->status, PAGE_SIZE); skb_shinfo(nskb)->nr_frags = 0; diff --git a/drivers/ssb/Kconfig b/drivers/ssb/Kconfig index c574dd210500..df30e1323252 100644 --- a/drivers/ssb/Kconfig +++ b/drivers/ssb/Kconfig @@ -89,27 +89,6 @@ config SSB_HOST_SOC If unsure, say N -config SSB_SILENT - bool "No SSB kernel messages" - depends on SSB && EXPERT - help - This option turns off all Sonics Silicon Backplane printks. - Note that you won't be able to identify problems, once - messages are turned off. - This might only be desired for production kernels on - embedded devices to reduce the kernel size. - - Say N - -config SSB_DEBUG - bool "SSB debugging" - depends on SSB && !SSB_SILENT - help - This turns on additional runtime checks and debugging - messages. Turn this on for SSB troubleshooting. - - If unsure, say N - config SSB_SERIAL bool depends on SSB diff --git a/drivers/ssb/b43_pci_bridge.c b/drivers/ssb/b43_pci_bridge.c index bed2fedeb057..9c7316b5685f 100644 --- a/drivers/ssb/b43_pci_bridge.c +++ b/drivers/ssb/b43_pci_bridge.c @@ -10,12 +10,12 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/pci.h> #include <linux/module.h> #include <linux/ssb/ssb.h> -#include "ssb_private.h" - static const struct pci_device_id b43_pci_bridge_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, 0x4301) }, diff --git a/drivers/ssb/bridge_pcmcia_80211.c b/drivers/ssb/bridge_pcmcia_80211.c index d70568ea02d5..f51f150307df 100644 --- a/drivers/ssb/bridge_pcmcia_80211.c +++ b/drivers/ssb/bridge_pcmcia_80211.c @@ -6,6 +6,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/slab.h> #include <linux/module.h> @@ -15,8 +17,6 @@ #include <pcmcia/ds.h> #include <pcmcia/cisreg.h> -#include "ssb_private.h" - static const struct pcmcia_device_id ssb_host_pcmcia_tbl[] = { PCMCIA_DEVICE_MANF_CARD(0x2D0, 0x448), PCMCIA_DEVICE_MANF_CARD(0x2D0, 0x476), @@ -70,7 +70,7 @@ err_disable: err_kfree_ssb: kfree(ssb); out_error: - ssb_err("Initialization failed (%d, %d)\n", res, err); + dev_err(&dev->dev, "Initialization failed (%d, %d)\n", res, err); return err; } diff --git a/drivers/ssb/driver_chipcommon.c b/drivers/ssb/driver_chipcommon.c index 7cb7d2c8fd86..99a4656d113d 100644 --- a/drivers/ssb/driver_chipcommon.c +++ b/drivers/ssb/driver_chipcommon.c @@ -9,14 +9,14 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/ssb/ssb_regs.h> #include <linux/export.h> #include <linux/pci.h> #include <linux/bcm47xx_wdt.h> -#include "ssb_private.h" - /* Clock sources */ enum ssb_clksrc { @@ -56,7 +56,7 @@ void ssb_chipco_set_clockmode(struct ssb_chipcommon *cc, if (cc->capabilities & SSB_CHIPCO_CAP_PMU) return; /* PMU controls clockmode, separated function needed */ - SSB_WARN_ON(ccdev->id.revision >= 20); + WARN_ON(ccdev->id.revision >= 20); /* chipcommon cores prior to rev6 don't support dynamic clock control */ if (ccdev->id.revision < 6) @@ -111,7 +111,7 @@ void ssb_chipco_set_clockmode(struct ssb_chipcommon *cc, } break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } @@ -164,7 +164,7 @@ static int chipco_pctl_clockfreqlimit(struct ssb_chipcommon *cc, int get_max) divisor = 32; break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } else if (cc->dev->id.revision < 10) { switch (clocksrc) { @@ -277,7 +277,7 @@ static void calc_fast_powerup_delay(struct ssb_chipcommon *cc) minfreq = chipco_pctl_clockfreqlimit(cc, 0); pll_on_delay = chipco_read32(cc, SSB_CHIPCO_PLLONDELAY); tmp = (((pll_on_delay + 2) * 1000000) + (minfreq - 1)) / minfreq; - SSB_WARN_ON(tmp & ~0xFFFF); + WARN_ON(tmp & ~0xFFFF); cc->fast_pwrup_delay = tmp; } @@ -354,7 +354,7 @@ void ssb_chipcommon_init(struct ssb_chipcommon *cc) if (cc->dev->id.revision >= 11) cc->status = chipco_read32(cc, SSB_CHIPCO_CHIPSTAT); - ssb_dbg("chipcommon status is 0x%x\n", cc->status); + dev_dbg(cc->dev->dev, "chipcommon status is 0x%x\n", cc->status); if (cc->dev->id.revision >= 20) { chipco_write32(cc, SSB_CHIPCO_GPIOPULLUP, 0); diff --git a/drivers/ssb/driver_chipcommon_pmu.c b/drivers/ssb/driver_chipcommon_pmu.c index c5352ea4821e..0f60e90ded26 100644 --- a/drivers/ssb/driver_chipcommon_pmu.c +++ b/drivers/ssb/driver_chipcommon_pmu.c @@ -8,6 +8,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/ssb/ssb_regs.h> #include <linux/ssb/ssb_driver_chipcommon.h> @@ -17,8 +19,6 @@ #include <linux/bcm47xx_nvram.h> #endif -#include "ssb_private.h" - static u32 ssb_chipco_pll_read(struct ssb_chipcommon *cc, u32 offset) { chipco_write32(cc, SSB_CHIPCO_PLLCTL_ADDR, offset); @@ -110,7 +110,7 @@ static void ssb_pmu0_pllinit_r0(struct ssb_chipcommon *cc, return; } - ssb_info("Programming PLL to %u.%03u MHz\n", + dev_info(cc->dev->dev, "Programming PLL to %u.%03u MHz\n", crystalfreq / 1000, crystalfreq % 1000); /* First turn the PLL off. */ @@ -128,7 +128,7 @@ static void ssb_pmu0_pllinit_r0(struct ssb_chipcommon *cc, ~(1 << SSB_PMURES_5354_BB_PLL_PU)); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } for (i = 1500; i; i--) { tmp = chipco_read32(cc, SSB_CHIPCO_CLKCTLST); @@ -138,7 +138,7 @@ static void ssb_pmu0_pllinit_r0(struct ssb_chipcommon *cc, } tmp = chipco_read32(cc, SSB_CHIPCO_CLKCTLST); if (tmp & SSB_CHIPCO_CLKCTLST_HAVEHT) - ssb_emerg("Failed to turn the PLL off!\n"); + dev_emerg(cc->dev->dev, "Failed to turn the PLL off!\n"); /* Set PDIV in PLL control 0. */ pllctl = ssb_chipco_pll_read(cc, SSB_PMU0_PLLCTL0); @@ -249,7 +249,7 @@ static void ssb_pmu1_pllinit_r0(struct ssb_chipcommon *cc, return; } - ssb_info("Programming PLL to %u.%03u MHz\n", + dev_info(cc->dev->dev, "Programming PLL to %u.%03u MHz\n", crystalfreq / 1000, crystalfreq % 1000); /* First turn the PLL off. */ @@ -265,7 +265,7 @@ static void ssb_pmu1_pllinit_r0(struct ssb_chipcommon *cc, buffer_strength = 0x222222; break; default: - SSB_WARN_ON(1); + WARN_ON(1); } for (i = 1500; i; i--) { tmp = chipco_read32(cc, SSB_CHIPCO_CLKCTLST); @@ -275,7 +275,7 @@ static void ssb_pmu1_pllinit_r0(struct ssb_chipcommon *cc, } tmp = chipco_read32(cc, SSB_CHIPCO_CLKCTLST); if (tmp & SSB_CHIPCO_CLKCTLST_HAVEHT) - ssb_emerg("Failed to turn the PLL off!\n"); + dev_emerg(cc->dev->dev, "Failed to turn the PLL off!\n"); /* Set p1div and p2div. */ pllctl = ssb_chipco_pll_read(cc, SSB_PMU1_PLLCTL0); @@ -349,7 +349,7 @@ static void ssb_pmu_pll_init(struct ssb_chipcommon *cc) case 43222: break; default: - ssb_err("ERROR: PLL init unknown for device %04X\n", + dev_err(cc->dev->dev, "ERROR: PLL init unknown for device %04X\n", bus->chip_id); } } @@ -471,7 +471,7 @@ static void ssb_pmu_resources_init(struct ssb_chipcommon *cc) max_msk = 0xFFFFF; break; default: - ssb_err("ERROR: PMU resource config unknown for device %04X\n", + dev_err(cc->dev->dev, "ERROR: PMU resource config unknown for device %04X\n", bus->chip_id); } @@ -501,7 +501,7 @@ static void ssb_pmu_resources_init(struct ssb_chipcommon *cc) ~(depend_tab[i].depend)); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } } @@ -524,7 +524,7 @@ void ssb_pmu_init(struct ssb_chipcommon *cc) pmucap = chipco_read32(cc, SSB_CHIPCO_PMU_CAP); cc->pmu.rev = (pmucap & SSB_CHIPCO_PMU_CAP_REVISION); - ssb_dbg("Found rev %u PMU (capabilities 0x%08X)\n", + dev_dbg(cc->dev->dev, "Found rev %u PMU (capabilities 0x%08X)\n", cc->pmu.rev, pmucap); if (cc->pmu.rev == 1) @@ -568,12 +568,12 @@ void ssb_pmu_set_ldo_voltage(struct ssb_chipcommon *cc, mask = 0x3F; break; default: - SSB_WARN_ON(1); + WARN_ON(1); return; } break; case 0x4312: - if (SSB_WARN_ON(id != LDO_PAREF)) + if (WARN_ON(id != LDO_PAREF)) return; addr = 0; shift = 21; @@ -636,7 +636,7 @@ u32 ssb_pmu_get_alp_clock(struct ssb_chipcommon *cc) case 0x5354: return ssb_pmu_get_alp_clock_clk0(cc); default: - ssb_err("ERROR: PMU alp clock unknown for device %04X\n", + dev_err(cc->dev->dev, "ERROR: PMU alp clock unknown for device %04X\n", bus->chip_id); return 0; } @@ -651,7 +651,7 @@ u32 ssb_pmu_get_cpu_clock(struct ssb_chipcommon *cc) /* 5354 chip uses a non programmable PLL of frequency 240MHz */ return 240000000; default: - ssb_err("ERROR: PMU cpu clock unknown for device %04X\n", + dev_err(cc->dev->dev, "ERROR: PMU cpu clock unknown for device %04X\n", bus->chip_id); return 0; } @@ -665,7 +665,7 @@ u32 ssb_pmu_get_controlclock(struct ssb_chipcommon *cc) case 0x5354: return 120000000; default: - ssb_err("ERROR: PMU controlclock unknown for device %04X\n", + dev_err(cc->dev->dev, "ERROR: PMU controlclock unknown for device %04X\n", bus->chip_id); return 0; } @@ -705,9 +705,9 @@ void ssb_pmu_spuravoid_pllupdate(struct ssb_chipcommon *cc, int spuravoid) pmu_ctl = SSB_CHIPCO_PMU_CTL_PLL_UPD; break; default: - ssb_printk(KERN_ERR PFX - "Unknown spuravoidance settings for chip 0x%04X, not changing PLL\n", - cc->dev->bus->chip_id); + dev_err(cc->dev->dev, + "Unknown spuravoidance settings for chip 0x%04X, not changing PLL\n", + cc->dev->bus->chip_id); return; } diff --git a/drivers/ssb/driver_chipcommon_sflash.c b/drivers/ssb/driver_chipcommon_sflash.c index 937fc31971a7..fac0e6828288 100644 --- a/drivers/ssb/driver_chipcommon_sflash.c +++ b/drivers/ssb/driver_chipcommon_sflash.c @@ -5,10 +5,10 @@ * Licensed under the GNU/GPL. See COPYING for details. */ -#include <linux/ssb/ssb.h> - #include "ssb_private.h" +#include <linux/ssb/ssb.h> + static struct resource ssb_sflash_resource = { .name = "ssb_sflash", .start = SSB_FLASH2, @@ -80,7 +80,7 @@ static void ssb_sflash_cmd(struct ssb_chipcommon *cc, u32 opcode) return; cpu_relax(); } - pr_err("SFLASH control command failed (timeout)!\n"); + dev_err(cc->dev->dev, "SFLASH control command failed (timeout)!\n"); } /* Initialize serial flash access */ diff --git a/drivers/ssb/driver_extif.c b/drivers/ssb/driver_extif.c index 59385fdab5b0..06b68dd6e022 100644 --- a/drivers/ssb/driver_extif.c +++ b/drivers/ssb/driver_extif.c @@ -10,12 +10,12 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/serial.h> #include <linux/serial_core.h> #include <linux/serial_reg.h> -#include "ssb_private.h" - static inline u32 extif_read32(struct ssb_extif *extif, u16 offset) { diff --git a/drivers/ssb/driver_gige.c b/drivers/ssb/driver_gige.c index e9734051e3c4..ebee6b0e3c34 100644 --- a/drivers/ssb/driver_gige.c +++ b/drivers/ssb/driver_gige.c @@ -242,7 +242,7 @@ static int ssb_gige_probe(struct ssb_device *sdev, bool pdev_is_ssb_gige_core(struct pci_dev *pdev) { if (!pdev->resource[0].name) - return 0; + return false; return (strcmp(pdev->resource[0].name, SSB_GIGE_MEM_RES_NAME) == 0); } EXPORT_SYMBOL(pdev_is_ssb_gige_core); diff --git a/drivers/ssb/driver_gpio.c b/drivers/ssb/driver_gpio.c index 796e22037bc4..e809dae4c470 100644 --- a/drivers/ssb/driver_gpio.c +++ b/drivers/ssb/driver_gpio.c @@ -8,6 +8,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/gpio/driver.h> #include <linux/irq.h> #include <linux/interrupt.h> @@ -15,8 +17,6 @@ #include <linux/export.h> #include <linux/ssb/ssb.h> -#include "ssb_private.h" - /************************************************** * Shared @@ -461,7 +461,7 @@ int ssb_gpio_init(struct ssb_bus *bus) else if (ssb_extif_available(&bus->extif)) return ssb_gpio_extif_init(bus); else - SSB_WARN_ON(1); + WARN_ON(1); return -1; } @@ -473,7 +473,7 @@ int ssb_gpio_unregister(struct ssb_bus *bus) gpiochip_remove(&bus->gpio); return 0; } else { - SSB_WARN_ON(1); + WARN_ON(1); } return -1; diff --git a/drivers/ssb/driver_mipscore.c b/drivers/ssb/driver_mipscore.c index f87efef42252..1ca2ac5ef2b8 100644 --- a/drivers/ssb/driver_mipscore.c +++ b/drivers/ssb/driver_mipscore.c @@ -8,6 +8,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/mtd/physmap.h> @@ -19,8 +21,6 @@ #include <linux/bcm47xx_nvram.h> #endif -#include "ssb_private.h" - static const char * const part_probes[] = { "bcm47xxpart", NULL }; static struct physmap_flash_data ssb_pflash_data = { @@ -170,14 +170,15 @@ static void set_irq(struct ssb_device *dev, unsigned int irq) irqflag |= (ipsflag & ~ipsflag_irq_mask[irq]); ssb_write32(mdev, SSB_IPSFLAG, irqflag); } - ssb_dbg("set_irq: core 0x%04x, irq %d => %d\n", + dev_dbg(dev->dev, "set_irq: core 0x%04x, irq %d => %d\n", dev->id.coreid, oldirq+2, irq+2); } static void print_irq(struct ssb_device *dev, unsigned int irq) { static const char *irq_name[] = {"2(S)", "3", "4", "5", "6", "D", "I"}; - ssb_dbg("core 0x%04x, irq : %s%s %s%s %s%s %s%s %s%s %s%s %s%s\n", + dev_dbg(dev->dev, + "core 0x%04x, irq : %s%s %s%s %s%s %s%s %s%s %s%s %s%s\n", dev->id.coreid, irq_name[0], irq == 0 ? "*" : " ", irq_name[1], irq == 1 ? "*" : " ", @@ -229,11 +230,11 @@ static void ssb_mips_flash_detect(struct ssb_mipscore *mcore) switch (bus->chipco.capabilities & SSB_CHIPCO_CAP_FLASHT) { case SSB_CHIPCO_FLASHT_STSER: case SSB_CHIPCO_FLASHT_ATSER: - pr_debug("Found serial flash\n"); + dev_dbg(mcore->dev->dev, "Found serial flash\n"); ssb_sflash_init(&bus->chipco); break; case SSB_CHIPCO_FLASHT_PARA: - pr_debug("Found parallel flash\n"); + dev_dbg(mcore->dev->dev, "Found parallel flash\n"); pflash->present = true; pflash->window = SSB_FLASH2; pflash->window_size = SSB_FLASH2_SZ; @@ -299,7 +300,7 @@ void ssb_mipscore_init(struct ssb_mipscore *mcore) if (!mcore->dev) return; /* We don't have a MIPS core */ - ssb_dbg("Initializing MIPS core...\n"); + dev_dbg(mcore->dev->dev, "Initializing MIPS core...\n"); bus = mcore->dev->bus; hz = ssb_clockspeed(bus); @@ -347,7 +348,7 @@ void ssb_mipscore_init(struct ssb_mipscore *mcore) break; } } - ssb_dbg("after irq reconfiguration\n"); + dev_dbg(mcore->dev->dev, "after irq reconfiguration\n"); dump_irq(bus); ssb_mips_serial_init(mcore); diff --git a/drivers/ssb/driver_pcicore.c b/drivers/ssb/driver_pcicore.c index 5fe1c22e289b..6a5622e0ded5 100644 --- a/drivers/ssb/driver_pcicore.c +++ b/drivers/ssb/driver_pcicore.c @@ -8,14 +8,14 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/pci.h> #include <linux/export.h> #include <linux/delay.h> #include <linux/ssb/ssb_embedded.h> -#include "ssb_private.h" - static u32 ssb_pcie_read(struct ssb_pcicore *pc, u32 address); static void ssb_pcie_write(struct ssb_pcicore *pc, u32 address, u32 data); static u16 ssb_pcie_mdio_read(struct ssb_pcicore *pc, u8 device, u8 address); @@ -115,7 +115,7 @@ static int ssb_extpci_read_config(struct ssb_pcicore *pc, u32 addr, val; void __iomem *mmio; - SSB_WARN_ON(!pc->hostmode); + WARN_ON(!pc->hostmode); if (unlikely(len != 1 && len != 2 && len != 4)) goto out; addr = get_cfgspace_addr(pc, bus, dev, func, off); @@ -161,7 +161,7 @@ static int ssb_extpci_write_config(struct ssb_pcicore *pc, u32 addr, val = 0; void __iomem *mmio; - SSB_WARN_ON(!pc->hostmode); + WARN_ON(!pc->hostmode); if (unlikely(len != 1 && len != 2 && len != 4)) goto out; addr = get_cfgspace_addr(pc, bus, dev, func, off); @@ -263,7 +263,7 @@ int ssb_pcicore_plat_dev_init(struct pci_dev *d) return -ENODEV; } - ssb_info("PCI: Fixing up device %s\n", pci_name(d)); + dev_info(&d->dev, "PCI: Fixing up device %s\n", pci_name(d)); /* Fix up interrupt lines */ d->irq = ssb_mips_irq(extpci_core->dev) + 2; @@ -284,12 +284,12 @@ static void ssb_pcicore_fixup_pcibridge(struct pci_dev *dev) if (dev->bus->number != 0 || PCI_SLOT(dev->devfn) != 0) return; - ssb_info("PCI: Fixing up bridge %s\n", pci_name(dev)); + dev_info(&dev->dev, "PCI: Fixing up bridge %s\n", pci_name(dev)); /* Enable PCI bridge bus mastering and memory space */ pci_set_master(dev); if (pcibios_enable_device(dev, ~0) < 0) { - ssb_err("PCI: SSB bridge enable failed\n"); + dev_err(&dev->dev, "PCI: SSB bridge enable failed\n"); return; } @@ -298,7 +298,8 @@ static void ssb_pcicore_fixup_pcibridge(struct pci_dev *dev) /* Make sure our latency is high enough to handle the devices behind us */ lat = 168; - ssb_info("PCI: Fixing latency timer of device %s to %u\n", + dev_info(&dev->dev, + "PCI: Fixing latency timer of device %s to %u\n", pci_name(dev), lat); pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat); } @@ -322,7 +323,7 @@ static void ssb_pcicore_init_hostmode(struct ssb_pcicore *pc) return; extpci_core = pc; - ssb_dbg("PCIcore in host mode found\n"); + dev_dbg(pc->dev->dev, "PCIcore in host mode found\n"); /* Reset devices on the external PCI bus */ val = SSB_PCICORE_CTL_RST_OE; val |= SSB_PCICORE_CTL_CLK_OE; @@ -337,7 +338,7 @@ static void ssb_pcicore_init_hostmode(struct ssb_pcicore *pc) udelay(1); /* Assertion time demanded by the PCI standard */ if (pc->dev->bus->has_cardbus_slot) { - ssb_dbg("CardBus slot detected\n"); + dev_dbg(pc->dev->dev, "CardBus slot detected\n"); pc->cardbusmode = 1; /* GPIO 1 resets the bridge */ ssb_gpio_out(pc->dev->bus, 1, 1); @@ -701,7 +702,7 @@ int ssb_pcicore_dev_irqvecs_enable(struct ssb_pcicore *pc, /* Calculate the "coremask" for the device. */ coremask = (1 << dev->core_index); - SSB_WARN_ON(bus->bustype != SSB_BUSTYPE_PCI); + WARN_ON(bus->bustype != SSB_BUSTYPE_PCI); err = pci_read_config_dword(bus->host_pci, SSB_PCI_IRQMASK, &tmp); if (err) goto out; diff --git a/drivers/ssb/embedded.c b/drivers/ssb/embedded.c index 55e101115038..8254ed25e063 100644 --- a/drivers/ssb/embedded.c +++ b/drivers/ssb/embedded.c @@ -9,6 +9,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/export.h> #include <linux/platform_device.h> #include <linux/ssb/ssb.h> @@ -17,8 +19,6 @@ #include <linux/ssb/ssb_driver_gige.h> #include <linux/pci.h> -#include "ssb_private.h" - int ssb_watchdog_timer_set(struct ssb_bus *bus, u32 ticks) { @@ -57,8 +57,8 @@ int ssb_watchdog_register(struct ssb_bus *bus) bus->busnumber, &wdt, sizeof(wdt)); if (IS_ERR(pdev)) { - ssb_dbg("can not register watchdog device, err: %li\n", - PTR_ERR(pdev)); + pr_debug("can not register watchdog device, err: %li\n", + PTR_ERR(pdev)); return PTR_ERR(pdev); } @@ -77,7 +77,7 @@ u32 ssb_gpio_in(struct ssb_bus *bus, u32 mask) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_in(&bus->extif, mask); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -95,7 +95,7 @@ u32 ssb_gpio_out(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_out(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -113,7 +113,7 @@ u32 ssb_gpio_outen(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_outen(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -145,7 +145,7 @@ u32 ssb_gpio_intmask(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_intmask(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -163,7 +163,7 @@ u32 ssb_gpio_polarity(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_polarity(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; diff --git a/drivers/ssb/host_soc.c b/drivers/ssb/host_soc.c index d62992dc08b2..3b438480515c 100644 --- a/drivers/ssb/host_soc.c +++ b/drivers/ssb/host_soc.c @@ -8,11 +8,11 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/bcm47xx_nvram.h> #include <linux/ssb/ssb.h> -#include "ssb_private.h" - static u8 ssb_host_soc_read8(struct ssb_device *dev, u16 offset) { struct ssb_bus *bus = dev->bus; @@ -61,7 +61,7 @@ static void ssb_host_soc_block_read(struct ssb_device *dev, void *buffer, case sizeof(u16): { __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { *buf = (__force __le16)__raw_readw(addr); buf++; @@ -72,7 +72,7 @@ static void ssb_host_soc_block_read(struct ssb_device *dev, void *buffer, case sizeof(u32): { __le32 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { *buf = (__force __le32)__raw_readl(addr); buf++; @@ -81,7 +81,7 @@ static void ssb_host_soc_block_read(struct ssb_device *dev, void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } } #endif /* CONFIG_SSB_BLOCKIO */ @@ -134,7 +134,7 @@ static void ssb_host_soc_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u16): { const __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { __raw_writew((__force u16)(*buf), addr); buf++; @@ -145,7 +145,7 @@ static void ssb_host_soc_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u32): { const __le32 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { __raw_writel((__force u32)(*buf), addr); buf++; @@ -154,7 +154,7 @@ static void ssb_host_soc_block_write(struct ssb_device *dev, const void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } } #endif /* CONFIG_SSB_BLOCKIO */ diff --git a/drivers/ssb/main.c b/drivers/ssb/main.c index 116594413f66..0a26984acb2c 100644 --- a/drivers/ssb/main.c +++ b/drivers/ssb/main.c @@ -209,7 +209,7 @@ int ssb_devices_freeze(struct ssb_bus *bus, struct ssb_freeze_context *ctx) memset(ctx, 0, sizeof(*ctx)); ctx->bus = bus; - SSB_WARN_ON(bus->nr_devices > ARRAY_SIZE(ctx->device_frozen)); + WARN_ON(bus->nr_devices > ARRAY_SIZE(ctx->device_frozen)); for (i = 0; i < bus->nr_devices; i++) { sdev = ssb_device_get(&bus->devices[i]); @@ -220,7 +220,7 @@ int ssb_devices_freeze(struct ssb_bus *bus, struct ssb_freeze_context *ctx) continue; } sdrv = drv_to_ssb_drv(sdev->dev->driver); - if (SSB_WARN_ON(!sdrv->remove)) + if (WARN_ON(!sdrv->remove)) continue; sdrv->remove(sdev); ctx->device_frozen[i] = 1; @@ -248,15 +248,16 @@ int ssb_devices_thaw(struct ssb_freeze_context *ctx) continue; sdev = &bus->devices[i]; - if (SSB_WARN_ON(!sdev->dev || !sdev->dev->driver)) + if (WARN_ON(!sdev->dev || !sdev->dev->driver)) continue; sdrv = drv_to_ssb_drv(sdev->dev->driver); - if (SSB_WARN_ON(!sdrv || !sdrv->probe)) + if (WARN_ON(!sdrv || !sdrv->probe)) continue; err = sdrv->probe(sdev, &sdev->id); if (err) { - ssb_err("Failed to thaw device %s\n", + dev_err(sdev->dev, + "Failed to thaw device %s\n", dev_name(sdev->dev)); result = err; } @@ -431,9 +432,9 @@ void ssb_bus_unregister(struct ssb_bus *bus) err = ssb_gpio_unregister(bus); if (err == -EBUSY) - ssb_dbg("Some GPIOs are still in use\n"); + pr_debug("Some GPIOs are still in use\n"); else if (err) - ssb_dbg("Can not unregister GPIO driver: %i\n", err); + pr_debug("Can not unregister GPIO driver: %i\n", err); ssb_buses_lock(); ssb_devices_unregister(bus); @@ -518,7 +519,7 @@ static int ssb_devices_register(struct ssb_bus *bus) sdev->dev = dev; err = device_register(dev); if (err) { - ssb_err("Could not register %s\n", dev_name(dev)); + pr_err("Could not register %s\n", dev_name(dev)); /* Set dev to NULL to not unregister * dev on error unwinding. */ sdev->dev = NULL; @@ -576,9 +577,9 @@ static int ssb_attach_queued_buses(void) err = ssb_gpio_init(bus); if (err == -ENOTSUPP) - ssb_dbg("GPIO driver not activated\n"); + pr_debug("GPIO driver not activated\n"); else if (err) - ssb_dbg("Error registering GPIO driver: %i\n", err); + pr_debug("Error registering GPIO driver: %i\n", err); ssb_bus_may_powerdown(bus); @@ -707,10 +708,12 @@ int ssb_bus_pcibus_register(struct ssb_bus *bus, struct pci_dev *host_pci) err = ssb_bus_register(bus, ssb_pci_get_invariants, 0); if (!err) { - ssb_info("Sonics Silicon Backplane found on PCI device %s\n", + dev_info(&host_pci->dev, + "Sonics Silicon Backplane found on PCI device %s\n", dev_name(&host_pci->dev)); } else { - ssb_err("Failed to register PCI version of SSB with error %d\n", + dev_err(&host_pci->dev, + "Failed to register PCI version of SSB with error %d\n", err); } @@ -731,7 +734,8 @@ int ssb_bus_pcmciabus_register(struct ssb_bus *bus, err = ssb_bus_register(bus, ssb_pcmcia_get_invariants, baseaddr); if (!err) { - ssb_info("Sonics Silicon Backplane found on PCMCIA device %s\n", + dev_info(&pcmcia_dev->dev, + "Sonics Silicon Backplane found on PCMCIA device %s\n", pcmcia_dev->devname); } @@ -752,7 +756,8 @@ int ssb_bus_sdiobus_register(struct ssb_bus *bus, struct sdio_func *func, err = ssb_bus_register(bus, ssb_sdio_get_invariants, ~0); if (!err) { - ssb_info("Sonics Silicon Backplane found on SDIO device %s\n", + dev_info(&func->dev, + "Sonics Silicon Backplane found on SDIO device %s\n", sdio_func_id(func)); } @@ -771,8 +776,8 @@ int ssb_bus_host_soc_register(struct ssb_bus *bus, unsigned long baseaddr) err = ssb_bus_register(bus, ssb_host_soc_get_invariants, baseaddr); if (!err) { - ssb_info("Sonics Silicon Backplane found at address 0x%08lX\n", - baseaddr); + pr_info("Sonics Silicon Backplane found at address 0x%08lX\n", + baseaddr); } return err; @@ -856,13 +861,13 @@ u32 ssb_calc_clock_rate(u32 plltype, u32 n, u32 m) case SSB_PLLTYPE_2: /* 48Mhz, 4 dividers */ n1 += SSB_CHIPCO_CLK_T2_BIAS; n2 += SSB_CHIPCO_CLK_T2_BIAS; - SSB_WARN_ON(!((n1 >= 2) && (n1 <= 7))); - SSB_WARN_ON(!((n2 >= 5) && (n2 <= 23))); + WARN_ON(!((n1 >= 2) && (n1 <= 7))); + WARN_ON(!((n2 >= 5) && (n2 <= 23))); break; case SSB_PLLTYPE_5: /* 25Mhz, 4 dividers */ return 100000000; default: - SSB_WARN_ON(1); + WARN_ON(1); } switch (plltype) { @@ -911,9 +916,9 @@ u32 ssb_calc_clock_rate(u32 plltype, u32 n, u32 m) m1 += SSB_CHIPCO_CLK_T2_BIAS; m2 += SSB_CHIPCO_CLK_T2M2_BIAS; m3 += SSB_CHIPCO_CLK_T2_BIAS; - SSB_WARN_ON(!((m1 >= 2) && (m1 <= 7))); - SSB_WARN_ON(!((m2 >= 3) && (m2 <= 10))); - SSB_WARN_ON(!((m3 >= 2) && (m3 <= 7))); + WARN_ON(!((m1 >= 2) && (m1 <= 7))); + WARN_ON(!((m2 >= 3) && (m2 <= 10))); + WARN_ON(!((m3 >= 2) && (m3 <= 7))); if (!(mc & SSB_CHIPCO_CLK_T2MC_M1BYP)) clock /= m1; @@ -923,7 +928,7 @@ u32 ssb_calc_clock_rate(u32 plltype, u32 n, u32 m) clock /= m3; return clock; default: - SSB_WARN_ON(1); + WARN_ON(1); } return 0; } @@ -1057,9 +1062,9 @@ static int ssb_wait_bits(struct ssb_device *dev, u16 reg, u32 bitmask, } udelay(10); } - printk(KERN_ERR PFX "Timeout waiting for bitmask %08X on " - "register %04X to %s.\n", - bitmask, reg, (set ? "set" : "clear")); + dev_err(dev->dev, + "Timeout waiting for bitmask %08X on register %04X to %s\n", + bitmask, reg, set ? "set" : "clear"); return -ETIMEDOUT; } @@ -1164,12 +1169,10 @@ int ssb_bus_may_powerdown(struct ssb_bus *bus) if (err) goto error; out: -#ifdef CONFIG_SSB_DEBUG bus->powered_up = 0; -#endif return err; error: - ssb_err("Bus powerdown failed\n"); + pr_err("Bus powerdown failed\n"); goto out; } EXPORT_SYMBOL(ssb_bus_may_powerdown); @@ -1183,16 +1186,14 @@ int ssb_bus_powerup(struct ssb_bus *bus, bool dynamic_pctl) if (err) goto error; -#ifdef CONFIG_SSB_DEBUG bus->powered_up = 1; -#endif mode = dynamic_pctl ? SSB_CLKMODE_DYNAMIC : SSB_CLKMODE_FAST; ssb_chipco_set_clockmode(&bus->chipco, mode); return 0; error: - ssb_err("Bus powerup failed\n"); + pr_err("Bus powerup failed\n"); return err; } EXPORT_SYMBOL(ssb_bus_powerup); @@ -1237,15 +1238,15 @@ u32 ssb_admatch_base(u32 adm) base = (adm & SSB_ADM_BASE0); break; case SSB_ADM_TYPE1: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ base = (adm & SSB_ADM_BASE1); break; case SSB_ADM_TYPE2: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ base = (adm & SSB_ADM_BASE2); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } return base; @@ -1261,15 +1262,15 @@ u32 ssb_admatch_size(u32 adm) size = ((adm & SSB_ADM_SZ0) >> SSB_ADM_SZ0_SHIFT); break; case SSB_ADM_TYPE1: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ size = ((adm & SSB_ADM_SZ1) >> SSB_ADM_SZ1_SHIFT); break; case SSB_ADM_TYPE2: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ size = ((adm & SSB_ADM_SZ2) >> SSB_ADM_SZ2_SHIFT); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } size = (1 << (size + 1)); @@ -1300,19 +1301,19 @@ static int __init ssb_modinit(void) err = b43_pci_ssb_bridge_init(); if (err) { - ssb_err("Broadcom 43xx PCI-SSB-bridge initialization failed\n"); + pr_err("Broadcom 43xx PCI-SSB-bridge initialization failed\n"); /* don't fail SSB init because of this */ err = 0; } err = ssb_host_pcmcia_init(); if (err) { - ssb_err("PCMCIA host initialization failed\n"); + pr_err("PCMCIA host initialization failed\n"); /* don't fail SSB init because of this */ err = 0; } err = ssb_gige_init(); if (err) { - ssb_err("SSB Broadcom Gigabit Ethernet driver initialization failed\n"); + pr_err("SSB Broadcom Gigabit Ethernet driver initialization failed\n"); /* don't fail SSB init because of this */ err = 0; } diff --git a/drivers/ssb/pci.c b/drivers/ssb/pci.c index 77b551da5728..84807a9b4b13 100644 --- a/drivers/ssb/pci.c +++ b/drivers/ssb/pci.c @@ -15,14 +15,14 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/ssb/ssb_regs.h> #include <linux/slab.h> #include <linux/pci.h> #include <linux/delay.h> -#include "ssb_private.h" - /* Define the following to 1 to enable a printk on each coreswitch. */ #define SSB_VERBOSE_PCICORESWITCH_DEBUG 0 @@ -56,7 +56,7 @@ int ssb_pci_switch_coreidx(struct ssb_bus *bus, u8 coreidx) } return 0; error: - ssb_err("Failed to switch to core %u\n", coreidx); + pr_err("Failed to switch to core %u\n", coreidx); return -ENODEV; } @@ -67,9 +67,8 @@ int ssb_pci_switch_core(struct ssb_bus *bus, unsigned long flags; #if SSB_VERBOSE_PCICORESWITCH_DEBUG - ssb_info("Switching to %s core, index %d\n", - ssb_core_name(dev->id.coreid), - dev->core_index); + pr_info("Switching to %s core, index %d\n", + ssb_core_name(dev->id.coreid), dev->core_index); #endif spin_lock_irqsave(&bus->bar_lock, flags); @@ -161,7 +160,7 @@ out: return err; err_pci: - printk(KERN_ERR PFX "Error: ssb_pci_xtal() could not access PCI config space!\n"); + pr_err("Error: ssb_pci_xtal() could not access PCI config space!\n"); err = -EBUSY; goto out; } @@ -286,7 +285,7 @@ static int sprom_do_write(struct ssb_bus *bus, const u16 *sprom) u32 spromctl; u16 size = bus->sprom_size; - ssb_notice("Writing SPROM. Do NOT turn off the power! Please stand by...\n"); + pr_notice("Writing SPROM. Do NOT turn off the power! Please stand by...\n"); err = pci_read_config_dword(pdev, SSB_SPROMCTL, &spromctl); if (err) goto err_ctlreg; @@ -294,17 +293,17 @@ static int sprom_do_write(struct ssb_bus *bus, const u16 *sprom) err = pci_write_config_dword(pdev, SSB_SPROMCTL, spromctl); if (err) goto err_ctlreg; - ssb_notice("[ 0%%"); + pr_notice("[ 0%%"); msleep(500); for (i = 0; i < size; i++) { if (i == size / 4) - ssb_cont("25%%"); + pr_cont("25%%"); else if (i == size / 2) - ssb_cont("50%%"); + pr_cont("50%%"); else if (i == (size * 3) / 4) - ssb_cont("75%%"); + pr_cont("75%%"); else if (i % 2) - ssb_cont("."); + pr_cont("."); writew(sprom[i], bus->mmio + bus->sprom_offset + (i * 2)); mmiowb(); msleep(20); @@ -317,12 +316,12 @@ static int sprom_do_write(struct ssb_bus *bus, const u16 *sprom) if (err) goto err_ctlreg; msleep(500); - ssb_cont("100%% ]\n"); - ssb_notice("SPROM written\n"); + pr_cont("100%% ]\n"); + pr_notice("SPROM written\n"); return 0; err_ctlreg: - ssb_err("Could not access SPROM control register.\n"); + pr_err("Could not access SPROM control register.\n"); return err; } @@ -816,7 +815,7 @@ static int sprom_extract(struct ssb_bus *bus, struct ssb_sprom *out, memset(out, 0, sizeof(*out)); out->revision = in[size - 1] & 0x00FF; - ssb_dbg("SPROM revision %d detected\n", out->revision); + pr_debug("SPROM revision %d detected\n", out->revision); memset(out->et0mac, 0xFF, 6); /* preset et0 and et1 mac */ memset(out->et1mac, 0xFF, 6); @@ -825,7 +824,7 @@ static int sprom_extract(struct ssb_bus *bus, struct ssb_sprom *out, * number stored in the SPROM. * Always extract r1. */ out->revision = 1; - ssb_dbg("SPROM treated as revision %d\n", out->revision); + pr_debug("SPROM treated as revision %d\n", out->revision); } switch (out->revision) { @@ -842,8 +841,8 @@ static int sprom_extract(struct ssb_bus *bus, struct ssb_sprom *out, sprom_extract_r8(out, in); break; default: - ssb_warn("Unsupported SPROM revision %d detected. Will extract v1\n", - out->revision); + pr_warn("Unsupported SPROM revision %d detected. Will extract v1\n", + out->revision); out->revision = 1; sprom_extract_r123(out, in); } @@ -863,7 +862,7 @@ static int ssb_pci_sprom_get(struct ssb_bus *bus, u16 *buf; if (!ssb_is_sprom_available(bus)) { - ssb_err("No SPROM available!\n"); + pr_err("No SPROM available!\n"); return -ENODEV; } if (bus->chipco.dev) { /* can be unavailable! */ @@ -882,7 +881,7 @@ static int ssb_pci_sprom_get(struct ssb_bus *bus, } else { bus->sprom_offset = SSB_SPROM_BASE1; } - ssb_dbg("SPROM offset is 0x%x\n", bus->sprom_offset); + pr_debug("SPROM offset is 0x%x\n", bus->sprom_offset); buf = kcalloc(SSB_SPROMSIZE_WORDS_R123, sizeof(u16), GFP_KERNEL); if (!buf) @@ -907,16 +906,16 @@ static int ssb_pci_sprom_get(struct ssb_bus *bus, * available for this device in some other storage */ err = ssb_fill_sprom_with_fallback(bus, sprom); if (err) { - ssb_warn("WARNING: Using fallback SPROM failed (err %d)\n", - err); + pr_warn("WARNING: Using fallback SPROM failed (err %d)\n", + err); goto out_free; } else { - ssb_dbg("Using SPROM revision %d provided by platform\n", - sprom->revision); + pr_debug("Using SPROM revision %d provided by platform\n", + sprom->revision); err = 0; goto out_free; } - ssb_warn("WARNING: Invalid SPROM CRC (corrupt SPROM)\n"); + pr_warn("WARNING: Invalid SPROM CRC (corrupt SPROM)\n"); } } err = sprom_extract(bus, sprom, buf, bus->sprom_size); @@ -947,14 +946,12 @@ out: return err; } -#ifdef CONFIG_SSB_DEBUG static int ssb_pci_assert_buspower(struct ssb_bus *bus) { if (likely(bus->powered_up)) return 0; - printk(KERN_ERR PFX "FATAL ERROR: Bus powered down " - "while accessing PCI MMIO space\n"); + pr_err("FATAL ERROR: Bus powered down while accessing PCI MMIO space\n"); if (bus->power_warn_count <= 10) { bus->power_warn_count++; dump_stack(); @@ -962,12 +959,6 @@ static int ssb_pci_assert_buspower(struct ssb_bus *bus) return -ENODEV; } -#else /* DEBUG */ -static inline int ssb_pci_assert_buspower(struct ssb_bus *bus) -{ - return 0; -} -#endif /* DEBUG */ static u8 ssb_pci_read8(struct ssb_device *dev, u16 offset) { @@ -1026,15 +1017,15 @@ static void ssb_pci_block_read(struct ssb_device *dev, void *buffer, ioread8_rep(addr, buffer, count); break; case sizeof(u16): - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); ioread16_rep(addr, buffer, count >> 1); break; case sizeof(u32): - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); ioread32_rep(addr, buffer, count >> 2); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } return; @@ -1100,15 +1091,15 @@ static void ssb_pci_block_write(struct ssb_device *dev, const void *buffer, iowrite8_rep(addr, buffer, count); break; case sizeof(u16): - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); iowrite16_rep(addr, buffer, count >> 1); break; case sizeof(u32): - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); iowrite32_rep(addr, buffer, count >> 2); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } #endif /* CONFIG_SSB_BLOCKIO */ diff --git a/drivers/ssb/pcmcia.c b/drivers/ssb/pcmcia.c index f03422bbf087..567013f8a8be 100644 --- a/drivers/ssb/pcmcia.c +++ b/drivers/ssb/pcmcia.c @@ -8,6 +8,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/delay.h> #include <linux/io.h> @@ -18,8 +20,6 @@ #include <pcmcia/ds.h> #include <pcmcia/cisreg.h> -#include "ssb_private.h" - /* Define the following to 1 to enable a printk on each coreswitch. */ #define SSB_VERBOSE_PCMCIACORESWITCH_DEBUG 0 @@ -143,7 +143,7 @@ int ssb_pcmcia_switch_coreidx(struct ssb_bus *bus, return 0; error: - ssb_err("Failed to switch to core %u\n", coreidx); + pr_err("Failed to switch to core %u\n", coreidx); return err; } @@ -152,9 +152,8 @@ static int ssb_pcmcia_switch_core(struct ssb_bus *bus, struct ssb_device *dev) int err; #if SSB_VERBOSE_PCMCIACORESWITCH_DEBUG - ssb_info("Switching to %s core, index %d\n", - ssb_core_name(dev->id.coreid), - dev->core_index); + pr_info("Switching to %s core, index %d\n", + ssb_core_name(dev->id.coreid), dev->core_index); #endif err = ssb_pcmcia_switch_coreidx(bus, dev->core_index); @@ -170,7 +169,7 @@ int ssb_pcmcia_switch_segment(struct ssb_bus *bus, u8 seg) int err; u8 val; - SSB_WARN_ON((seg != 0) && (seg != 1)); + WARN_ON((seg != 0) && (seg != 1)); while (1) { err = ssb_pcmcia_cfg_write(bus, SSB_PCMCIA_MEMSEG, seg); if (err) @@ -190,7 +189,7 @@ int ssb_pcmcia_switch_segment(struct ssb_bus *bus, u8 seg) return 0; error: - ssb_err("Failed to switch pcmcia segment\n"); + pr_err("Failed to switch pcmcia segment\n"); return err; } @@ -300,7 +299,7 @@ static void ssb_pcmcia_block_read(struct ssb_device *dev, void *buffer, case sizeof(u16): { __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { *buf = (__force __le16)__raw_readw(addr); buf++; @@ -311,7 +310,7 @@ static void ssb_pcmcia_block_read(struct ssb_device *dev, void *buffer, case sizeof(u32): { __le16 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { *buf = (__force __le16)__raw_readw(addr); buf++; @@ -322,7 +321,7 @@ static void ssb_pcmcia_block_read(struct ssb_device *dev, void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } unlock: spin_unlock_irqrestore(&bus->bar_lock, flags); @@ -400,7 +399,7 @@ static void ssb_pcmcia_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u16): { const __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { __raw_writew((__force u16)(*buf), addr); buf++; @@ -411,7 +410,7 @@ static void ssb_pcmcia_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u32): { const __le16 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { __raw_writew((__force u16)(*buf), addr); buf++; @@ -422,7 +421,7 @@ static void ssb_pcmcia_block_write(struct ssb_device *dev, const void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } unlock: mmiowb(); @@ -547,39 +546,39 @@ static int ssb_pcmcia_sprom_write_all(struct ssb_bus *bus, const u16 *sprom) bool failed = 0; size_t size = SSB_PCMCIA_SPROM_SIZE; - ssb_notice("Writing SPROM. Do NOT turn off the power! Please stand by...\n"); + pr_notice("Writing SPROM. Do NOT turn off the power! Please stand by...\n"); err = ssb_pcmcia_sprom_command(bus, SSB_PCMCIA_SPROMCTL_WRITEEN); if (err) { - ssb_notice("Could not enable SPROM write access\n"); + pr_notice("Could not enable SPROM write access\n"); return -EBUSY; } - ssb_notice("[ 0%%"); + pr_notice("[ 0%%"); msleep(500); for (i = 0; i < size; i++) { if (i == size / 4) - ssb_cont("25%%"); + pr_cont("25%%"); else if (i == size / 2) - ssb_cont("50%%"); + pr_cont("50%%"); else if (i == (size * 3) / 4) - ssb_cont("75%%"); + pr_cont("75%%"); else if (i % 2) - ssb_cont("."); + pr_cont("."); err = ssb_pcmcia_sprom_write(bus, i, sprom[i]); if (err) { - ssb_notice("Failed to write to SPROM\n"); + pr_notice("Failed to write to SPROM\n"); failed = 1; break; } } err = ssb_pcmcia_sprom_command(bus, SSB_PCMCIA_SPROMCTL_WRITEDIS); if (err) { - ssb_notice("Could not disable SPROM write access\n"); + pr_notice("Could not disable SPROM write access\n"); failed = 1; } msleep(500); if (!failed) { - ssb_cont("100%% ]\n"); - ssb_notice("SPROM written\n"); + pr_cont("100%% ]\n"); + pr_notice("SPROM written\n"); } return failed ? -EBUSY : 0; @@ -693,9 +692,8 @@ static int ssb_pcmcia_do_get_invariants(struct pcmcia_device *p_dev, return -ENOSPC; /* continue with next entry */ error: - ssb_err( - "PCMCIA: Failed to fetch device invariants: %s\n", - error_description); + pr_err("PCMCIA: Failed to fetch device invariants: %s\n", + error_description); return -ENODEV; } @@ -715,8 +713,7 @@ int ssb_pcmcia_get_invariants(struct ssb_bus *bus, res = pcmcia_loop_tuple(bus->host_pcmcia, CISTPL_FUNCE, ssb_pcmcia_get_mac, sprom); if (res != 0) { - ssb_err( - "PCMCIA: Failed to fetch MAC address\n"); + pr_err("PCMCIA: Failed to fetch MAC address\n"); return -ENODEV; } @@ -726,8 +723,7 @@ int ssb_pcmcia_get_invariants(struct ssb_bus *bus, if ((res == 0) || (res == -ENOSPC)) return 0; - ssb_err( - "PCMCIA: Failed to fetch device invariants\n"); + pr_err("PCMCIA: Failed to fetch device invariants\n"); return -ENODEV; } @@ -836,6 +832,6 @@ int ssb_pcmcia_init(struct ssb_bus *bus) return 0; error: - ssb_err("Failed to initialize PCMCIA host device\n"); + pr_err("Failed to initialize PCMCIA host device\n"); return err; } diff --git a/drivers/ssb/scan.c b/drivers/ssb/scan.c index b9429df583eb..6ceee98ed6ff 100644 --- a/drivers/ssb/scan.c +++ b/drivers/ssb/scan.c @@ -12,6 +12,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/ssb/ssb_regs.h> #include <linux/pci.h> @@ -20,8 +22,6 @@ #include <pcmcia/cistpl.h> #include <pcmcia/ds.h> -#include "ssb_private.h" - const char *ssb_core_name(u16 coreid) { @@ -125,7 +125,7 @@ static u16 pcidev_to_chipid(struct pci_dev *pci_dev) chipid_fallback = 0x4401; break; default: - ssb_err("PCI-ID not in fallback list\n"); + dev_err(&pci_dev->dev, "PCI-ID not in fallback list\n"); } return chipid_fallback; @@ -151,7 +151,7 @@ static u8 chipid_to_nrcores(u16 chipid) case 0x4704: return 9; default: - ssb_err("CHIPID not in nrcores fallback list\n"); + pr_err("CHIPID not in nrcores fallback list\n"); } return 1; @@ -210,7 +210,7 @@ void ssb_iounmap(struct ssb_bus *bus) #ifdef CONFIG_SSB_PCIHOST pci_iounmap(bus->host_pci, bus->mmio); #else - SSB_BUG_ON(1); /* Can't reach this code. */ + WARN_ON(1); /* Can't reach this code. */ #endif break; case SSB_BUSTYPE_SDIO: @@ -236,7 +236,7 @@ static void __iomem *ssb_ioremap(struct ssb_bus *bus, #ifdef CONFIG_SSB_PCIHOST mmio = pci_iomap(bus->host_pci, 0, ~0UL); #else - SSB_BUG_ON(1); /* Can't reach this code. */ + WARN_ON(1); /* Can't reach this code. */ #endif break; case SSB_BUSTYPE_SDIO: @@ -318,13 +318,13 @@ int ssb_bus_scan(struct ssb_bus *bus, bus->chip_package = 0; } } - ssb_info("Found chip with id 0x%04X, rev 0x%02X and package 0x%02X\n", - bus->chip_id, bus->chip_rev, bus->chip_package); + pr_info("Found chip with id 0x%04X, rev 0x%02X and package 0x%02X\n", + bus->chip_id, bus->chip_rev, bus->chip_package); if (!bus->nr_devices) bus->nr_devices = chipid_to_nrcores(bus->chip_id); if (bus->nr_devices > ARRAY_SIZE(bus->devices)) { - ssb_err("More than %d ssb cores found (%d)\n", - SSB_MAX_NR_CORES, bus->nr_devices); + pr_err("More than %d ssb cores found (%d)\n", + SSB_MAX_NR_CORES, bus->nr_devices); goto err_unmap; } if (bus->bustype == SSB_BUSTYPE_SSB) { @@ -355,18 +355,16 @@ int ssb_bus_scan(struct ssb_bus *bus, dev->bus = bus; dev->ops = bus->ops; - printk(KERN_DEBUG PFX - "Core %d found: %s " - "(cc 0x%03X, rev 0x%02X, vendor 0x%04X)\n", - i, ssb_core_name(dev->id.coreid), - dev->id.coreid, dev->id.revision, dev->id.vendor); + pr_debug("Core %d found: %s (cc 0x%03X, rev 0x%02X, vendor 0x%04X)\n", + i, ssb_core_name(dev->id.coreid), + dev->id.coreid, dev->id.revision, dev->id.vendor); switch (dev->id.coreid) { case SSB_DEV_80211: nr_80211_cores++; if (nr_80211_cores > 1) { if (!we_support_multiple_80211_cores(bus)) { - ssb_dbg("Ignoring additional 802.11 core\n"); + pr_debug("Ignoring additional 802.11 core\n"); continue; } } @@ -374,7 +372,7 @@ int ssb_bus_scan(struct ssb_bus *bus, case SSB_DEV_EXTIF: #ifdef CONFIG_SSB_DRIVER_EXTIF if (bus->extif.dev) { - ssb_warn("WARNING: Multiple EXTIFs found\n"); + pr_warn("WARNING: Multiple EXTIFs found\n"); break; } bus->extif.dev = dev; @@ -382,7 +380,7 @@ int ssb_bus_scan(struct ssb_bus *bus, break; case SSB_DEV_CHIPCOMMON: if (bus->chipco.dev) { - ssb_warn("WARNING: Multiple ChipCommon found\n"); + pr_warn("WARNING: Multiple ChipCommon found\n"); break; } bus->chipco.dev = dev; @@ -391,7 +389,7 @@ int ssb_bus_scan(struct ssb_bus *bus, case SSB_DEV_MIPS_3302: #ifdef CONFIG_SSB_DRIVER_MIPS if (bus->mipscore.dev) { - ssb_warn("WARNING: Multiple MIPS cores found\n"); + pr_warn("WARNING: Multiple MIPS cores found\n"); break; } bus->mipscore.dev = dev; @@ -412,7 +410,7 @@ int ssb_bus_scan(struct ssb_bus *bus, } } if (bus->pcicore.dev) { - ssb_warn("WARNING: Multiple PCI(E) cores found\n"); + pr_warn("WARNING: Multiple PCI(E) cores found\n"); break; } bus->pcicore.dev = dev; diff --git a/drivers/ssb/sdio.c b/drivers/ssb/sdio.c index 2278e43614bd..7fe0afb42234 100644 --- a/drivers/ssb/sdio.c +++ b/drivers/ssb/sdio.c @@ -12,14 +12,14 @@ * */ +#include "ssb_private.h" + #include <linux/ssb/ssb.h> #include <linux/delay.h> #include <linux/io.h> #include <linux/etherdevice.h> #include <linux/mmc/sdio_func.h> -#include "ssb_private.h" - /* Define the following to 1 to enable a printk on each coreswitch. */ #define SSB_VERBOSE_SDIOCORESWITCH_DEBUG 0 @@ -316,18 +316,18 @@ static void ssb_sdio_block_read(struct ssb_device *dev, void *buffer, break; } case sizeof(u16): { - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); error = sdio_readsb(bus->host_sdio, buffer, offset, count); break; } case sizeof(u32): { - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); offset |= SBSDIO_SB_ACCESS_2_4B_FLAG; /* 32 bit data access */ error = sdio_readsb(bus->host_sdio, buffer, offset, count); break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } if (!error) goto out; @@ -423,18 +423,18 @@ static void ssb_sdio_block_write(struct ssb_device *dev, const void *buffer, (void *)buffer, count); break; case sizeof(u16): - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); error = sdio_writesb(bus->host_sdio, offset, (void *)buffer, count); break; case sizeof(u32): - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); offset |= SBSDIO_SB_ACCESS_2_4B_FLAG; /* 32 bit data access */ error = sdio_writesb(bus->host_sdio, offset, (void *)buffer, count); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } if (!error) goto out; diff --git a/drivers/ssb/sprom.c b/drivers/ssb/sprom.c index e753fbe302a7..4f028a80d6c4 100644 --- a/drivers/ssb/sprom.c +++ b/drivers/ssb/sprom.c @@ -127,13 +127,13 @@ ssize_t ssb_attr_sprom_store(struct ssb_bus *bus, goto out_kfree; err = ssb_devices_freeze(bus, &freeze); if (err) { - ssb_err("SPROM write: Could not freeze all devices\n"); + pr_err("SPROM write: Could not freeze all devices\n"); goto out_unlock; } res = sprom_write(bus, sprom); err = ssb_devices_thaw(&freeze); if (err) - ssb_err("SPROM write: Could not thaw all devices\n"); + pr_err("SPROM write: Could not thaw all devices\n"); out_unlock: mutex_unlock(&bus->sprom_mutex); out_kfree: diff --git a/drivers/ssb/ssb_private.h b/drivers/ssb/ssb_private.h index ef9ac8efcab4..5f31bdfbe77f 100644 --- a/drivers/ssb/ssb_private.h +++ b/drivers/ssb/ssb_private.h @@ -2,47 +2,14 @@ #ifndef LINUX_SSB_PRIVATE_H_ #define LINUX_SSB_PRIVATE_H_ +#define PFX "ssb: " +#define pr_fmt(fmt) PFX fmt + #include <linux/ssb/ssb.h> #include <linux/types.h> #include <linux/bcm47xx_wdt.h> -#define PFX "ssb: " - -#ifdef CONFIG_SSB_SILENT -# define ssb_printk(fmt, ...) \ - do { if (0) printk(fmt, ##__VA_ARGS__); } while (0) -#else -# define ssb_printk(fmt, ...) \ - printk(fmt, ##__VA_ARGS__) -#endif /* CONFIG_SSB_SILENT */ - -#define ssb_emerg(fmt, ...) ssb_printk(KERN_EMERG PFX fmt, ##__VA_ARGS__) -#define ssb_err(fmt, ...) ssb_printk(KERN_ERR PFX fmt, ##__VA_ARGS__) -#define ssb_warn(fmt, ...) ssb_printk(KERN_WARNING PFX fmt, ##__VA_ARGS__) -#define ssb_notice(fmt, ...) ssb_printk(KERN_NOTICE PFX fmt, ##__VA_ARGS__) -#define ssb_info(fmt, ...) ssb_printk(KERN_INFO PFX fmt, ##__VA_ARGS__) -#define ssb_cont(fmt, ...) ssb_printk(KERN_CONT fmt, ##__VA_ARGS__) - -/* dprintk: Debugging printk; vanishes for non-debug compilation */ -#ifdef CONFIG_SSB_DEBUG -# define ssb_dbg(fmt, ...) \ - ssb_printk(KERN_DEBUG PFX fmt, ##__VA_ARGS__) -#else -# define ssb_dbg(fmt, ...) \ - do { if (0) printk(KERN_DEBUG PFX fmt, ##__VA_ARGS__); } while (0) -#endif - -#ifdef CONFIG_SSB_DEBUG -# define SSB_WARN_ON(x) WARN_ON(x) -# define SSB_BUG_ON(x) BUG_ON(x) -#else -static inline int __ssb_do_nothing(int x) { return x; } -# define SSB_WARN_ON(x) __ssb_do_nothing(unlikely(!!(x))) -# define SSB_BUG_ON(x) __ssb_do_nothing(unlikely(!!(x))) -#endif - - /* pci.c */ #ifdef CONFIG_SSB_PCIHOST extern int ssb_pci_switch_core(struct ssb_bus *bus, diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 705aebd74e56..465a6f5142cc 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -421,7 +421,7 @@ const char *vp_bus_name(struct virtio_device *vdev) * - OR over all affinities for shared MSI * - ignore the affinity request if we're using INTX */ -int vp_set_vq_affinity(struct virtqueue *vq, int cpu) +int vp_set_vq_affinity(struct virtqueue *vq, const struct cpumask *cpu_mask) { struct virtio_device *vdev = vq->vdev; struct virtio_pci_device *vp_dev = to_vp_device(vdev); @@ -435,11 +435,10 @@ int vp_set_vq_affinity(struct virtqueue *vq, int cpu) if (vp_dev->msix_enabled) { mask = vp_dev->msix_affinity_masks[info->msix_vector]; irq = pci_irq_vector(vp_dev->pci_dev, info->msix_vector); - if (cpu == -1) + if (!cpu_mask) irq_set_affinity_hint(irq, NULL); else { - cpumask_clear(mask); - cpumask_set_cpu(cpu, mask); + cpumask_copy(mask, cpu_mask); irq_set_affinity_hint(irq, mask); } } diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index 135ee3cf7175..02271002c2f3 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -141,7 +141,7 @@ const char *vp_bus_name(struct virtio_device *vdev); * - OR over all affinities for shared MSI * - ignore the affinity request if we're using INTX */ -int vp_set_vq_affinity(struct virtqueue *vq, int cpu); +int vp_set_vq_affinity(struct virtqueue *vq, const struct cpumask *cpu_mask); const struct cpumask *vp_get_vq_affinity(struct virtio_device *vdev, int index); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index cd8790d2c6ed..523481a3471b 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -23,7 +23,7 @@ struct bpf_prog; struct bpf_map; struct sock; struct seq_file; -struct btf; +struct btf_type; /* map is generic key/value storage optionally accesible by eBPF programs */ struct bpf_map_ops { @@ -48,8 +48,9 @@ struct bpf_map_ops { u32 (*map_fd_sys_lookup_elem)(void *ptr); void (*map_seq_show_elem)(struct bpf_map *map, void *key, struct seq_file *m); - int (*map_check_btf)(const struct bpf_map *map, const struct btf *btf, - u32 key_type_id, u32 value_type_id); + int (*map_check_btf)(const struct bpf_map *map, + const struct btf_type *key_type, + const struct btf_type *value_type); }; struct bpf_map { @@ -118,9 +119,13 @@ static inline bool bpf_map_offload_neutral(const struct bpf_map *map) static inline bool bpf_map_support_seq_show(const struct bpf_map *map) { - return map->ops->map_seq_show_elem && map->ops->map_check_btf; + return map->btf && map->ops->map_seq_show_elem; } +int map_check_no_btf(const struct bpf_map *map, + const struct btf_type *key_type, + const struct btf_type *value_type); + extern const struct bpf_map_ops bpf_map_offload_ops; /* function argument constraints */ @@ -524,6 +529,7 @@ static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) } struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); +int array_map_alloc_check(union bpf_attr *attr); #else /* !CONFIG_BPF_SYSCALL */ static inline struct bpf_prog *bpf_prog_get(u32 ufd) @@ -769,6 +775,33 @@ static inline void __xsk_map_flush(struct bpf_map *map) } #endif +#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) +void bpf_sk_reuseport_detach(struct sock *sk); +int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, + void *value); +int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, + void *value, u64 map_flags); +#else +static inline void bpf_sk_reuseport_detach(struct sock *sk) +{ +} + +#ifdef CONFIG_BPF_SYSCALL +static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, + void *key, void *value) +{ + return -EOPNOTSUPP; +} + +static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, + void *key, void *value, + u64 map_flags) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_BPF_SYSCALL */ +#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ + /* verifier prototypes for helper functions called from eBPF programs */ extern const struct bpf_func_proto bpf_map_lookup_elem_proto; extern const struct bpf_func_proto bpf_map_update_elem_proto; diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index add08be53b6f..cd26c090e7c0 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -29,6 +29,9 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_DEVICE, cg_dev) #ifdef CONFIG_BPF_LIRC_MODE2 BPF_PROG_TYPE(BPF_PROG_TYPE_LIRC_MODE2, lirc_mode2) #endif +#ifdef CONFIG_INET +BPF_PROG_TYPE(BPF_PROG_TYPE_SK_REUSEPORT, sk_reuseport) +#endif BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) @@ -60,4 +63,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops) #if defined(CONFIG_XDP_SOCKETS) BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops) #endif +#ifdef CONFIG_INET +BPF_MAP_TYPE(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, reuseport_array_ops) +#endif #endif diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index c9fdf6f57913..32c553556bbd 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -554,6 +554,36 @@ static inline bool cgroup_is_descendant(struct cgroup *cgrp, } /** + * cgroup_ancestor - find ancestor of cgroup + * @cgrp: cgroup to find ancestor of + * @ancestor_level: level of ancestor to find starting from root + * + * Find ancestor of cgroup at specified level starting from root if it exists + * and return pointer to it. Return NULL if @cgrp doesn't have ancestor at + * @ancestor_level. + * + * This function is safe to call as long as @cgrp is accessible. + */ +static inline struct cgroup *cgroup_ancestor(struct cgroup *cgrp, + int ancestor_level) +{ + struct cgroup *ptr; + + if (cgrp->level < ancestor_level) + return NULL; + + for (ptr = cgrp; + ptr && ptr->level > ancestor_level; + ptr = cgroup_parent(ptr)) + ; + + if (ptr && ptr->level == ancestor_level) + return ptr; + + return NULL; +} + +/** * task_under_cgroup_hierarchy - test task's membership of cgroup ancestry * @task: the task to be tested * @ancestor: possible ancestor of @task's cgroup diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 57f20a0a7794..147bdec42215 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -159,6 +159,13 @@ static inline unsigned int cpumask_next_and(int n, return n+1; } +static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, + int start, bool wrap) +{ + /* cpu0 unless stop condition, wrap and at cpu0, then nr_cpumask_bits */ + return (wrap && n == 0); +} + /* cpu must be a valid cpu, ie 0, so there's no other choice. */ static inline unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) diff --git a/include/linux/filter.h b/include/linux/filter.h index c73dd7396886..5d565c50bcb2 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -32,6 +32,7 @@ struct seccomp_data; struct bpf_prog_aux; struct xdp_rxq_info; struct xdp_buff; +struct sock_reuseport; /* ArgX, context and stack frame pointer register positions. Note, * Arg1, Arg2, Arg3, etc are used as argument mappings of function @@ -537,6 +538,20 @@ struct sk_msg_buff { struct list_head list; }; +struct bpf_redirect_info { + u32 ifindex; + u32 flags; + struct bpf_map *map; + struct bpf_map *map_to_flush; + unsigned long map_owner; + u32 kern_flags; +}; + +DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info); + +/* flags for bpf_redirect_info kern_flags */ +#define BPF_RI_F_RF_NO_DIRECT BIT(0) /* no napi_direct on return_frame */ + /* Compute the linear packet data range [data, data_end) which * will be accessed by various program types (cls_bpf, act_bpf, * lwt, ...). Subsystems allowing direct data access must (!) @@ -738,6 +753,7 @@ int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk); int sk_attach_bpf(u32 ufd, struct sock *sk); int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk); int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk); +void sk_reuseport_prog_free(struct bpf_prog *prog); int sk_detach_filter(struct sock *sk); int sk_get_filter(struct sock *sk, struct sock_filter __user *filter, unsigned int len); @@ -765,6 +781,27 @@ static inline bool bpf_dump_raw_ok(void) struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, const struct bpf_insn *patch, u32 len); +static inline bool xdp_return_frame_no_direct(void) +{ + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + + return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT; +} + +static inline void xdp_set_return_frame_no_direct(void) +{ + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + + ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT; +} + +static inline void xdp_clear_return_frame_no_direct(void) +{ + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + + ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT; +} + static inline int xdp_ok_fwd_dev(const struct net_device *fwd, unsigned int pktlen) { @@ -798,6 +835,20 @@ void bpf_warn_invalid_xdp_action(u32 act); struct sock *do_sk_redirect_map(struct sk_buff *skb); struct sock *do_msg_redirect_map(struct sk_msg_buff *md); +#ifdef CONFIG_INET +struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, + struct bpf_prog *prog, struct sk_buff *skb, + u32 hash); +#else +static inline struct sock * +bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, + struct bpf_prog *prog, struct sk_buff *skb, + u32 hash) +{ + return NULL; +} +#endif + #ifdef CONFIG_BPF_JIT extern int bpf_jit_enable; extern int bpf_jit_harden; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 7e237a63a70c..17a13e4785fc 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1038,6 +1038,7 @@ static inline struct sk_buff *alloc_skb_fclone(unsigned int size, } struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src); +void skb_headers_offset_update(struct sk_buff *skb, int off); int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask); struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority); void skb_copy_header(struct sk_buff *new, const struct sk_buff *old); diff --git a/include/linux/ssb/ssb.h b/include/linux/ssb/ssb.h index 3b43655cabe6..0d5a2691e7e9 100644 --- a/include/linux/ssb/ssb.h +++ b/include/linux/ssb/ssb.h @@ -499,11 +499,9 @@ struct ssb_bus { /* Internal-only stuff follows. Do not touch. */ struct list_head list; -#ifdef CONFIG_SSB_DEBUG /* Is the bus already powered up? */ bool powered_up; int power_warn_count; -#endif /* DEBUG */ }; enum ssb_quirks { diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 5559a2d31c46..32baf8e26735 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -79,7 +79,8 @@ struct virtio_config_ops { u64 (*get_features)(struct virtio_device *vdev); int (*finalize_features)(struct virtio_device *vdev); const char *(*bus_name)(struct virtio_device *vdev); - int (*set_vq_affinity)(struct virtqueue *vq, int cpu); + int (*set_vq_affinity)(struct virtqueue *vq, + const struct cpumask *cpu_mask); const struct cpumask *(*get_vq_affinity)(struct virtio_device *vdev, int index); }; @@ -236,11 +237,11 @@ const char *virtio_bus_name(struct virtio_device *vdev) * */ static inline -int virtqueue_set_affinity(struct virtqueue *vq, int cpu) +int virtqueue_set_affinity(struct virtqueue *vq, const struct cpumask *cpu_mask) { struct virtio_device *vdev = vq->vdev; if (vdev->config->set_vq_affinity) - return vdev->config->set_vq_affinity(vq, cpu); + return vdev->config->set_vq_affinity(vq, cpu_mask); return 0; } diff --git a/include/net/act_api.h b/include/net/act_api.h index 8c9bc02d05e1..1ad5b19e83a9 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -101,6 +101,7 @@ struct tc_action_ops { void (*stats_update)(struct tc_action *, u64, u32, u64); size_t (*get_fill_size)(const struct tc_action *act); struct net_device *(*get_dev)(const struct tc_action *a); + void (*put_dev)(struct net_device *dev); int (*delete)(struct net *net, u32 index); }; diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 5f43f7a70fe6..6def0351bcc3 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -108,6 +108,7 @@ int ipv6_get_lladdr(struct net_device *dev, struct in6_addr *addr, u32 banned_flags); bool inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2, bool match_wildcard); +bool inet_rcv_saddr_any(const struct sock *sk); void addrconf_join_solict(struct net_device *dev, const struct in6_addr *addr); void addrconf_leave_solict(struct inet6_dev *idev, const struct in6_addr *addr); diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h index 0304ba2ae353..883bb9085f15 100644 --- a/include/net/gen_stats.h +++ b/include/net/gen_stats.h @@ -59,13 +59,13 @@ int gnet_stats_finish_copy(struct gnet_dump *d); int gen_new_estimator(struct gnet_stats_basic_packed *bstats, struct gnet_stats_basic_cpu __percpu *cpu_bstats, struct net_rate_estimator __rcu **rate_est, - spinlock_t *stats_lock, + spinlock_t *lock, seqcount_t *running, struct nlattr *opt); void gen_kill_estimator(struct net_rate_estimator __rcu **ptr); int gen_replace_estimator(struct gnet_stats_basic_packed *bstats, struct gnet_stats_basic_cpu __percpu *cpu_bstats, struct net_rate_estimator __rcu **ptr, - spinlock_t *stats_lock, + spinlock_t *lock, seqcount_t *running, struct nlattr *opt); bool gen_estimator_active(struct net_rate_estimator __rcu **ptr); bool gen_estimator_read(struct net_rate_estimator __rcu **ptr, diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 0a6c9e0f2b5a..fa43b82607d9 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -167,7 +167,8 @@ enum inet_csk_ack_state_t { ICSK_ACK_SCHED = 1, ICSK_ACK_TIMER = 2, ICSK_ACK_PUSHED = 4, - ICSK_ACK_PUSHED2 = 8 + ICSK_ACK_PUSHED2 = 8, + ICSK_ACK_NOW = 16 /* Send the next ACK immediately (once) */ }; void inet_csk_init_xmit_timers(struct sock *sk, diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h index b86d14528188..1662cbc0b46b 100644 --- a/include/net/inet_frag.h +++ b/include/net/inet_frag.h @@ -57,7 +57,9 @@ struct frag_v6_compare_key { * @lock: spinlock protecting this frag * @refcnt: reference count of the queue * @fragments: received fragments head + * @rb_fragments: received fragments rb-tree root * @fragments_tail: received fragments tail + * @last_run_head: the head of the last "run". see ip_fragment.c * @stamp: timestamp of the last received fragment * @len: total length of the original datagram * @meat: length of received fragments so far @@ -78,6 +80,7 @@ struct inet_frag_queue { struct sk_buff *fragments; /* Used in IPv6. */ struct rb_root rb_fragments; /* Used in IPv4. */ struct sk_buff *fragments_tail; + struct sk_buff *last_run_head; ktime_t stamp; int len; int meat; @@ -113,6 +116,9 @@ void inet_frag_kill(struct inet_frag_queue *q); void inet_frag_destroy(struct inet_frag_queue *q); struct inet_frag_queue *inet_frag_find(struct netns_frags *nf, void *key); +/* Free all skbs in the queue; return the sum of their truesizes. */ +unsigned int inet_frag_rbtree_purge(struct rb_root *root); + static inline void inet_frag_put(struct inet_frag_queue *q) { if (refcount_dec_and_test(&q->refcnt)) diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h index 762ac9931b62..f0e396ab9bec 100644 --- a/include/net/netns/ipv6.h +++ b/include/net/netns/ipv6.h @@ -32,6 +32,7 @@ struct netns_sysctl_ipv6 { int flowlabel_consistency; int auto_flowlabels; int icmpv6_time; + int icmpv6_echo_ignore_all; int anycast_src_echo_reply; int ip_nonlocal_bind; int fwmark_reflect; diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index ab869e0d8326..28a7c8e44636 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -57,6 +57,7 @@ #include <linux/atomic.h> /* This gets us atomic counters. */ #include <linux/skbuff.h> /* We need sk_buff_head. */ #include <linux/workqueue.h> /* We need tq_struct. */ +#include <linux/flex_array.h> /* We need flex_array. */ #include <linux/sctp.h> /* We need sctp* header structs. */ #include <net/sctp/auth.h> /* We need auth specific structs */ #include <net/ip.h> /* For inet_skb_parm */ @@ -398,37 +399,35 @@ void sctp_stream_update(struct sctp_stream *stream, struct sctp_stream *new); /* What is the current SSN number for this stream? */ #define sctp_ssn_peek(stream, type, sid) \ - ((stream)->type[sid].ssn) + (sctp_stream_##type((stream), (sid))->ssn) /* Return the next SSN number for this stream. */ #define sctp_ssn_next(stream, type, sid) \ - ((stream)->type[sid].ssn++) + (sctp_stream_##type((stream), (sid))->ssn++) /* Skip over this ssn and all below. */ #define sctp_ssn_skip(stream, type, sid, ssn) \ - ((stream)->type[sid].ssn = ssn + 1) + (sctp_stream_##type((stream), (sid))->ssn = ssn + 1) /* What is the current MID number for this stream? */ #define sctp_mid_peek(stream, type, sid) \ - ((stream)->type[sid].mid) + (sctp_stream_##type((stream), (sid))->mid) /* Return the next MID number for this stream. */ #define sctp_mid_next(stream, type, sid) \ - ((stream)->type[sid].mid++) + (sctp_stream_##type((stream), (sid))->mid++) /* Skip over this mid and all below. */ #define sctp_mid_skip(stream, type, sid, mid) \ - ((stream)->type[sid].mid = mid + 1) - -#define sctp_stream_in(asoc, sid) (&(asoc)->stream.in[sid]) + (sctp_stream_##type((stream), (sid))->mid = mid + 1) /* What is the current MID_uo number for this stream? */ #define sctp_mid_uo_peek(stream, type, sid) \ - ((stream)->type[sid].mid_uo) + (sctp_stream_##type((stream), (sid))->mid_uo) /* Return the next MID_uo number for this stream. */ #define sctp_mid_uo_next(stream, type, sid) \ - ((stream)->type[sid].mid_uo++) + (sctp_stream_##type((stream), (sid))->mid_uo++) /* * Pointers to address related SCTP functions. @@ -1440,8 +1439,8 @@ struct sctp_stream_in { }; struct sctp_stream { - struct sctp_stream_out *out; - struct sctp_stream_in *in; + struct flex_array *out; + struct flex_array *in; __u16 outcnt; __u16 incnt; /* Current stream being sent, if any */ @@ -1463,6 +1462,23 @@ struct sctp_stream { struct sctp_stream_interleave *si; }; +static inline struct sctp_stream_out *sctp_stream_out( + const struct sctp_stream *stream, + __u16 sid) +{ + return flex_array_get(stream->out, sid); +} + +static inline struct sctp_stream_in *sctp_stream_in( + const struct sctp_stream *stream, + __u16 sid) +{ + return flex_array_get(stream->in, sid); +} + +#define SCTP_SO(s, i) sctp_stream_out((s), (i)) +#define SCTP_SI(s, i) sctp_stream_in((s), (i)) + #define SCTP_STREAM_CLOSED 0x00 #define SCTP_STREAM_OPEN 0x01 diff --git a/include/net/sock_reuseport.h b/include/net/sock_reuseport.h index 0054b3a9b923..8a5f70c7cdf2 100644 --- a/include/net/sock_reuseport.h +++ b/include/net/sock_reuseport.h @@ -5,25 +5,36 @@ #include <linux/filter.h> #include <linux/skbuff.h> #include <linux/types.h> +#include <linux/spinlock.h> #include <net/sock.h> +extern spinlock_t reuseport_lock; + struct sock_reuseport { struct rcu_head rcu; u16 max_socks; /* length of socks */ u16 num_socks; /* elements in socks */ + /* The last synq overflow event timestamp of this + * reuse->socks[] group. + */ + unsigned int synq_overflow_ts; + /* ID stays the same even after the size of socks[] grows. */ + unsigned int reuseport_id; + bool bind_inany; struct bpf_prog __rcu *prog; /* optional BPF sock selector */ struct sock *socks[0]; /* array of sock pointers */ }; -extern int reuseport_alloc(struct sock *sk); -extern int reuseport_add_sock(struct sock *sk, struct sock *sk2); +extern int reuseport_alloc(struct sock *sk, bool bind_inany); +extern int reuseport_add_sock(struct sock *sk, struct sock *sk2, + bool bind_inany); extern void reuseport_detach_sock(struct sock *sk); extern struct sock *reuseport_select_sock(struct sock *sk, u32 hash, struct sk_buff *skb, int hdr_len); -extern struct bpf_prog *reuseport_attach_prog(struct sock *sk, - struct bpf_prog *prog); +extern int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog); +int reuseport_get_id(struct sock_reuseport *reuse); #endif /* _SOCK_REUSEPORT_H */ diff --git a/include/net/tcp.h b/include/net/tcp.h index d769dc20359b..d196901c9dba 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -36,6 +36,7 @@ #include <net/inet_hashtables.h> #include <net/checksum.h> #include <net/request_sock.h> +#include <net/sock_reuseport.h> #include <net/sock.h> #include <net/snmp.h> #include <net/ip.h> @@ -473,9 +474,22 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); */ static inline void tcp_synq_overflow(const struct sock *sk) { - unsigned int last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; + unsigned int last_overflow; unsigned int now = jiffies; + if (sk->sk_reuseport) { + struct sock_reuseport *reuse; + + reuse = rcu_dereference(sk->sk_reuseport_cb); + if (likely(reuse)) { + last_overflow = READ_ONCE(reuse->synq_overflow_ts); + if (time_after32(now, last_overflow + HZ)) + WRITE_ONCE(reuse->synq_overflow_ts, now); + return; + } + } + + last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; if (time_after32(now, last_overflow + HZ)) tcp_sk(sk)->rx_opt.ts_recent_stamp = now; } @@ -483,9 +497,21 @@ static inline void tcp_synq_overflow(const struct sock *sk) /* syncookies: no recent synqueue overflow on this listening socket? */ static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) { - unsigned int last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; + unsigned int last_overflow; unsigned int now = jiffies; + if (sk->sk_reuseport) { + struct sock_reuseport *reuse; + + reuse = rcu_dereference(sk->sk_reuseport_cb); + if (likely(reuse)) { + last_overflow = READ_ONCE(reuse->synq_overflow_ts); + return time_after32(now, last_overflow + + TCP_SYNCOOKIE_VALID); + } + } + + last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; return time_after32(now, last_overflow + TCP_SYNCOOKIE_VALID); } diff --git a/include/net/tls.h b/include/net/tls.h index d8b3b6578c01..d5c683e8bb22 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -124,10 +124,6 @@ struct tls_sw_context_rx { struct sk_buff *recv_pkt; u8 control; bool decrypted; - - char rx_aad_ciphertext[TLS_AAD_SPACE_SIZE]; - char rx_aad_plaintext[TLS_AAD_SPACE_SIZE]; - }; struct tls_record_info { diff --git a/include/net/xdp.h b/include/net/xdp.h index fcb033f51d8c..76b95256c266 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -84,6 +84,13 @@ struct xdp_frame { struct net_device *dev_rx; /* used by cpumap */ }; +/* Clear kernel pointers in xdp_frame */ +static inline void xdp_scrub_frame(struct xdp_frame *frame) +{ + frame->data = NULL; + frame->dev_rx = NULL; +} + /* Convert xdp_buff to xdp_frame */ static inline struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) diff --git a/include/trace/events/fib.h b/include/trace/events/fib.h index 9763cddd0594..6271bab63bfb 100644 --- a/include/trace/events/fib.h +++ b/include/trace/events/fib.h @@ -22,6 +22,7 @@ TRACE_EVENT(fib_table_lookup, __field( int, err ) __field( int, oif ) __field( int, iif ) + __field( u8, proto ) __field( __u8, tos ) __field( __u8, scope ) __field( __u8, flags ) @@ -31,7 +32,6 @@ TRACE_EVENT(fib_table_lookup, __array( __u8, saddr, 4 ) __field( u16, sport ) __field( u16, dport ) - __field( u8, proto ) __dynamic_array(char, name, IFNAMSIZ ) ), diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index dd5758dc35d3..66917a4eba27 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -126,6 +126,7 @@ enum bpf_map_type { BPF_MAP_TYPE_XSKMAP, BPF_MAP_TYPE_SOCKHASH, BPF_MAP_TYPE_CGROUP_STORAGE, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, }; enum bpf_prog_type { @@ -150,6 +151,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_CGROUP_SOCK_ADDR, BPF_PROG_TYPE_LWT_SEG6LOCAL, BPF_PROG_TYPE_LIRC_MODE2, + BPF_PROG_TYPE_SK_REUSEPORT, }; enum bpf_attach_type { @@ -2091,6 +2093,24 @@ union bpf_attr { * Return * The id is returned or 0 in case the id could not be retrieved. * + * u64 bpf_skb_ancestor_cgroup_id(struct sk_buff *skb, int ancestor_level) + * Description + * Return id of cgroup v2 that is ancestor of cgroup associated + * with the *skb* at the *ancestor_level*. The root cgroup is at + * *ancestor_level* zero and each step down the hierarchy + * increments the level. If *ancestor_level* == level of cgroup + * associated with *skb*, then return value will be same as that + * of **bpf_skb_cgroup_id**\ (). + * + * The helper is useful to implement policies based on cgroups + * that are upper in hierarchy than immediate cgroup associated + * with *skb*. + * + * The format of returned id and helper limitations are same as in + * **bpf_skb_cgroup_id**\ (). + * Return + * The id is returned or 0 in case the id could not be retrieved. + * * u64 bpf_get_current_cgroup_id(void) * Return * A 64-bit integer containing the current cgroup id based @@ -2113,6 +2133,14 @@ union bpf_attr { * the shared data. * Return * Pointer to the local storage area. + * + * int bpf_sk_select_reuseport(struct sk_reuseport_md *reuse, struct bpf_map *map, void *key, u64 flags) + * Description + * Select a SO_REUSEPORT sk from a BPF_MAP_TYPE_REUSEPORT_ARRAY map + * It checks the selected sk is matching the incoming + * request in the skb. + * Return + * 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -2196,7 +2224,9 @@ union bpf_attr { FN(rc_keydown), \ FN(skb_cgroup_id), \ FN(get_current_cgroup_id), \ - FN(get_local_storage), + FN(get_local_storage), \ + FN(sk_select_reuseport), \ + FN(skb_ancestor_cgroup_id), /* integer value in 'imm' field of BPF_CALL instruction selects which helper * function eBPF program intends to call @@ -2413,6 +2443,30 @@ struct sk_msg_md { __u32 local_port; /* stored in host byte order */ }; +struct sk_reuseport_md { + /* + * Start of directly accessible data. It begins from + * the tcp/udp header. + */ + void *data; + void *data_end; /* End of directly accessible data */ + /* + * Total length of packet (starting from the tcp/udp header). + * Note that the directly accessible bytes (data_end - data) + * could be less than this "len". Those bytes could be + * indirectly read by a helper "bpf_skb_load_bytes()". + */ + __u32 len; + /* + * Eth protocol in the mac header (network byte order). e.g. + * ETH_P_IP(0x0800) and ETH_P_IPV6(0x86DD) + */ + __u32 eth_protocol; + __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ + __u32 bind_inany; /* Is sock bound to an INANY address? */ + __u32 hash; /* A hash of the packet 4 tuples */ +}; + #define BPF_TAG_SIZE 8 struct bpf_prog_info { diff --git a/include/uapi/linux/ppp-ioctl.h b/include/uapi/linux/ppp-ioctl.h index 784c2e3e572e..88b5f9990320 100644 --- a/include/uapi/linux/ppp-ioctl.h +++ b/include/uapi/linux/ppp-ioctl.h @@ -68,7 +68,7 @@ struct ppp_option_data { struct pppol2tp_ioc_stats { __u16 tunnel_id; /* redundant */ __u16 session_id; /* if zero, get tunnel stats */ - __u32 using_ipsec:1; /* valid only for session_id == 0 */ + __u32 using_ipsec:1; __aligned_u64 tx_packets; __aligned_u64 tx_bytes; __aligned_u64 tx_errors; diff --git a/include/uapi/linux/sysctl.h b/include/uapi/linux/sysctl.h index 6b58371b1f0d..d71013fffaf6 100644 --- a/include/uapi/linux/sysctl.h +++ b/include/uapi/linux/sysctl.h @@ -575,7 +575,8 @@ enum { /* /proc/sys/net/ipv6/icmp */ enum { - NET_IPV6_ICMP_RATELIMIT=1 + NET_IPV6_ICMP_RATELIMIT = 1, + NET_IPV6_ICMP_ECHO_IGNORE_ALL = 2 }; /* /proc/sys/net/<protocol>/neigh/<dev> */ diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index e8906cbad81f..0488b8258321 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -23,3 +23,6 @@ ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif obj-$(CONFIG_CGROUP_BPF) += cgroup.o +ifeq ($(CONFIG_INET),y) +obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o +endif diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 2aa55d030c77..0c17aab3ce5f 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -54,7 +54,7 @@ static int bpf_array_alloc_percpu(struct bpf_array *array) } /* Called from syscall */ -static int array_map_alloc_check(union bpf_attr *attr) +int array_map_alloc_check(union bpf_attr *attr) { bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; int numa_node = bpf_map_attr_numa_node(attr); @@ -358,27 +358,20 @@ static void array_map_seq_show_elem(struct bpf_map *map, void *key, rcu_read_unlock(); } -static int array_map_check_btf(const struct bpf_map *map, const struct btf *btf, - u32 btf_key_id, u32 btf_value_id) +static int array_map_check_btf(const struct bpf_map *map, + const struct btf_type *key_type, + const struct btf_type *value_type) { - const struct btf_type *key_type, *value_type; - u32 key_size, value_size; u32 int_data; - key_type = btf_type_id_size(btf, &btf_key_id, &key_size); - if (!key_type || BTF_INFO_KIND(key_type->info) != BTF_KIND_INT) + if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT) return -EINVAL; int_data = *(u32 *)(key_type + 1); - /* bpf array can only take a u32 key. This check makes - * sure that the btf matches the attr used during map_create. + /* bpf array can only take a u32 key. This check makes sure + * that the btf matches the attr used during map_create. */ - if (BTF_INT_BITS(int_data) != 32 || key_size != 4 || - BTF_INT_OFFSET(int_data)) - return -EINVAL; - - value_type = btf_type_id_size(btf, &btf_value_id, &value_size); - if (!value_type || value_size != map->value_size) + if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data)) return -EINVAL; return 0; @@ -405,6 +398,7 @@ const struct bpf_map_ops percpu_array_map_ops = { .map_lookup_elem = percpu_array_map_lookup_elem, .map_update_elem = array_map_update_elem, .map_delete_elem = array_map_delete_elem, + .map_check_btf = array_map_check_btf, }; static int fd_array_map_alloc_check(union bpf_attr *attr) @@ -546,6 +540,7 @@ const struct bpf_map_ops prog_array_map_ops = { .map_fd_put_ptr = prog_fd_array_put_ptr, .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem, .map_release_uref = bpf_fd_array_map_clear, + .map_check_btf = map_check_no_btf, }; static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, @@ -634,6 +629,7 @@ const struct bpf_map_ops perf_event_array_map_ops = { .map_fd_get_ptr = perf_event_fd_array_get_ptr, .map_fd_put_ptr = perf_event_fd_array_put_ptr, .map_release = perf_event_fd_array_release, + .map_check_btf = map_check_no_btf, }; #ifdef CONFIG_CGROUPS @@ -665,6 +661,7 @@ const struct bpf_map_ops cgroup_array_map_ops = { .map_delete_elem = fd_array_map_delete_elem, .map_fd_get_ptr = cgroup_fd_array_get_ptr, .map_fd_put_ptr = cgroup_fd_array_put_ptr, + .map_check_btf = map_check_no_btf, }; #endif @@ -749,4 +746,5 @@ const struct bpf_map_ops array_of_maps_map_ops = { .map_fd_put_ptr = bpf_map_fd_put_ptr, .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem, .map_gen_lookup = array_of_map_gen_lookup, + .map_check_btf = map_check_no_btf, }; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index e0918d180f08..620bc5024d7d 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -69,7 +69,7 @@ struct bpf_cpu_map { }; static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu, - struct xdp_bulk_queue *bq); + struct xdp_bulk_queue *bq, bool in_napi_ctx); static u64 cpu_map_bitmap_size(const union bpf_attr *attr) { @@ -375,7 +375,7 @@ static void __cpu_map_entry_free(struct rcu_head *rcu) struct xdp_bulk_queue *bq = per_cpu_ptr(rcpu->bulkq, cpu); /* No concurrent bq_enqueue can run at this point */ - bq_flush_to_queue(rcpu, bq); + bq_flush_to_queue(rcpu, bq, false); } free_percpu(rcpu->bulkq); /* Cannot kthread_stop() here, last put free rcpu resources */ @@ -555,10 +555,11 @@ const struct bpf_map_ops cpu_map_ops = { .map_update_elem = cpu_map_update_elem, .map_lookup_elem = cpu_map_lookup_elem, .map_get_next_key = cpu_map_get_next_key, + .map_check_btf = map_check_no_btf, }; static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu, - struct xdp_bulk_queue *bq) + struct xdp_bulk_queue *bq, bool in_napi_ctx) { unsigned int processed = 0, drops = 0; const int to_cpu = rcpu->cpu; @@ -578,7 +579,10 @@ static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu, err = __ptr_ring_produce(q, xdpf); if (err) { drops++; - xdp_return_frame_rx_napi(xdpf); + if (likely(in_napi_ctx)) + xdp_return_frame_rx_napi(xdpf); + else + xdp_return_frame(xdpf); } processed++; } @@ -598,7 +602,7 @@ static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq); if (unlikely(bq->count == CPU_MAP_BULK_SIZE)) - bq_flush_to_queue(rcpu, bq); + bq_flush_to_queue(rcpu, bq, true); /* Notice, xdp_buff/page MUST be queued here, long enough for * driver to code invoking us to finished, due to driver @@ -661,7 +665,7 @@ void __cpu_map_flush(struct bpf_map *map) /* Flush all frames in bulkq to real queue */ bq = this_cpu_ptr(rcpu->bulkq); - bq_flush_to_queue(rcpu, bq); + bq_flush_to_queue(rcpu, bq, true); /* If already running, costs spin_lock_irqsave + smb_mb */ wake_up_process(rcpu->kthread); diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index d361fc1e3bf3..ac1df79f3788 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -217,7 +217,8 @@ void __dev_map_insert_ctx(struct bpf_map *map, u32 bit) } static int bq_xmit_all(struct bpf_dtab_netdev *obj, - struct xdp_bulk_queue *bq, u32 flags) + struct xdp_bulk_queue *bq, u32 flags, + bool in_napi_ctx) { struct net_device *dev = obj->dev; int sent = 0, drops = 0, err = 0; @@ -254,7 +255,10 @@ error: struct xdp_frame *xdpf = bq->q[i]; /* RX path under NAPI protection, can return frames faster */ - xdp_return_frame_rx_napi(xdpf); + if (likely(in_napi_ctx)) + xdp_return_frame_rx_napi(xdpf); + else + xdp_return_frame(xdpf); drops++; } goto out; @@ -286,7 +290,7 @@ void __dev_map_flush(struct bpf_map *map) __clear_bit(bit, bitmap); bq = this_cpu_ptr(dev->bulkq); - bq_xmit_all(dev, bq, XDP_XMIT_FLUSH); + bq_xmit_all(dev, bq, XDP_XMIT_FLUSH, true); } } @@ -316,7 +320,7 @@ static int bq_enqueue(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf, struct xdp_bulk_queue *bq = this_cpu_ptr(obj->bulkq); if (unlikely(bq->count == DEV_MAP_BULK_SIZE)) - bq_xmit_all(obj, bq, 0); + bq_xmit_all(obj, bq, 0, true); /* Ingress dev_rx will be the same for all xdp_frame's in * bulk_queue, because bq stored per-CPU and must be flushed @@ -385,7 +389,7 @@ static void dev_map_flush_old(struct bpf_dtab_netdev *dev) __clear_bit(dev->bit, bitmap); bq = per_cpu_ptr(dev->bulkq, cpu); - bq_xmit_all(dev, bq, XDP_XMIT_FLUSH); + bq_xmit_all(dev, bq, XDP_XMIT_FLUSH, false); } } } @@ -484,6 +488,7 @@ const struct bpf_map_ops dev_map_ops = { .map_lookup_elem = dev_map_lookup_elem, .map_update_elem = dev_map_update_elem, .map_delete_elem = dev_map_delete_elem, + .map_check_btf = map_check_no_btf, }; static int dev_map_notification(struct notifier_block *notifier, diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 513d9dfcf4ee..04b8eda94e7d 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -11,9 +11,11 @@ * General Public License for more details. */ #include <linux/bpf.h> +#include <linux/btf.h> #include <linux/jhash.h> #include <linux/filter.h> #include <linux/rculist_nulls.h> +#include <uapi/linux/btf.h> #include "percpu_freelist.h" #include "bpf_lru_list.h" #include "map_in_map.h" @@ -1162,6 +1164,27 @@ static void htab_map_free(struct bpf_map *map) kfree(htab); } +static void htab_map_seq_show_elem(struct bpf_map *map, void *key, + struct seq_file *m) +{ + void *value; + + rcu_read_lock(); + + value = htab_map_lookup_elem(map, key); + if (!value) { + rcu_read_unlock(); + return; + } + + btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); + seq_puts(m, ": "); + btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); + seq_puts(m, "\n"); + + rcu_read_unlock(); +} + const struct bpf_map_ops htab_map_ops = { .map_alloc_check = htab_map_alloc_check, .map_alloc = htab_map_alloc, @@ -1171,6 +1194,7 @@ const struct bpf_map_ops htab_map_ops = { .map_update_elem = htab_map_update_elem, .map_delete_elem = htab_map_delete_elem, .map_gen_lookup = htab_map_gen_lookup, + .map_seq_show_elem = htab_map_seq_show_elem, }; const struct bpf_map_ops htab_lru_map_ops = { @@ -1182,6 +1206,7 @@ const struct bpf_map_ops htab_lru_map_ops = { .map_update_elem = htab_lru_map_update_elem, .map_delete_elem = htab_lru_map_delete_elem, .map_gen_lookup = htab_lru_map_gen_lookup, + .map_seq_show_elem = htab_map_seq_show_elem, }; /* Called from eBPF program */ @@ -1408,4 +1433,5 @@ const struct bpf_map_ops htab_of_maps_map_ops = { .map_fd_put_ptr = bpf_map_fd_put_ptr, .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem, .map_gen_lookup = htab_of_map_gen_lookup, + .map_check_btf = map_check_no_btf, }; diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 76efe9a183f5..2ada5e21dfa6 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -196,19 +196,21 @@ static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos) { struct bpf_map *map = seq_file_to_map(m); void *key = map_iter(m)->key; + void *prev_key; if (map_iter(m)->done) return NULL; if (unlikely(v == SEQ_START_TOKEN)) - goto done; + prev_key = NULL; + else + prev_key = key; - if (map->ops->map_get_next_key(map, key, key)) { + if (map->ops->map_get_next_key(map, prev_key, key)) { map_iter(m)->done = true; return NULL; } -done: ++(*pos); return key; } @@ -332,7 +334,8 @@ static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg) struct bpf_map *map = arg; return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops, - map->btf ? &bpffs_map_fops : &bpffs_obj_fops); + bpf_map_support_seq_show(map) ? + &bpffs_map_fops : &bpffs_obj_fops); } static struct dentry * diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index fc4e37f68f2a..22ad967d1e5f 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -246,6 +246,7 @@ const struct bpf_map_ops cgroup_storage_map_ops = { .map_lookup_elem = cgroup_storage_lookup_elem, .map_update_elem = cgroup_storage_update_elem, .map_delete_elem = cgroup_storage_delete_elem, + .map_check_btf = map_check_no_btf, }; int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map) diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c index 1603492c9cc7..9058317ba9de 100644 --- a/kernel/bpf/lpm_trie.c +++ b/kernel/bpf/lpm_trie.c @@ -10,11 +10,13 @@ */ #include <linux/bpf.h> +#include <linux/btf.h> #include <linux/err.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/vmalloc.h> #include <net/ipv6.h> +#include <uapi/linux/btf.h> /* Intermediate node */ #define LPM_TREE_NODE_FLAG_IM BIT(0) @@ -686,6 +688,15 @@ free_stack: return err; } +static int trie_check_btf(const struct bpf_map *map, + const struct btf_type *key_type, + const struct btf_type *value_type) +{ + /* Keys must have struct bpf_lpm_trie_key embedded. */ + return BTF_INFO_KIND(key_type->info) != BTF_KIND_STRUCT ? + -EINVAL : 0; +} + const struct bpf_map_ops trie_map_ops = { .map_alloc = trie_alloc, .map_free = trie_free, @@ -693,4 +704,5 @@ const struct bpf_map_ops trie_map_ops = { .map_lookup_elem = trie_lookup_elem, .map_update_elem = trie_update_elem, .map_delete_elem = trie_delete_elem, + .map_check_btf = trie_check_btf, }; diff --git a/kernel/bpf/reuseport_array.c b/kernel/bpf/reuseport_array.c new file mode 100644 index 000000000000..18e225de80ff --- /dev/null +++ b/kernel/bpf/reuseport_array.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Facebook + */ +#include <linux/bpf.h> +#include <linux/err.h> +#include <linux/sock_diag.h> +#include <net/sock_reuseport.h> + +struct reuseport_array { + struct bpf_map map; + struct sock __rcu *ptrs[]; +}; + +static struct reuseport_array *reuseport_array(struct bpf_map *map) +{ + return (struct reuseport_array *)map; +} + +/* The caller must hold the reuseport_lock */ +void bpf_sk_reuseport_detach(struct sock *sk) +{ + struct sock __rcu **socks; + + write_lock_bh(&sk->sk_callback_lock); + socks = sk->sk_user_data; + if (socks) { + WRITE_ONCE(sk->sk_user_data, NULL); + /* + * Do not move this NULL assignment outside of + * sk->sk_callback_lock because there is + * a race with reuseport_array_free() + * which does not hold the reuseport_lock. + */ + RCU_INIT_POINTER(*socks, NULL); + } + write_unlock_bh(&sk->sk_callback_lock); +} + +static int reuseport_array_alloc_check(union bpf_attr *attr) +{ + if (attr->value_size != sizeof(u32) && + attr->value_size != sizeof(u64)) + return -EINVAL; + + return array_map_alloc_check(attr); +} + +static void *reuseport_array_lookup_elem(struct bpf_map *map, void *key) +{ + struct reuseport_array *array = reuseport_array(map); + u32 index = *(u32 *)key; + + if (unlikely(index >= array->map.max_entries)) + return NULL; + + return rcu_dereference(array->ptrs[index]); +} + +/* Called from syscall only */ +static int reuseport_array_delete_elem(struct bpf_map *map, void *key) +{ + struct reuseport_array *array = reuseport_array(map); + u32 index = *(u32 *)key; + struct sock *sk; + int err; + + if (index >= map->max_entries) + return -E2BIG; + + if (!rcu_access_pointer(array->ptrs[index])) + return -ENOENT; + + spin_lock_bh(&reuseport_lock); + + sk = rcu_dereference_protected(array->ptrs[index], + lockdep_is_held(&reuseport_lock)); + if (sk) { + write_lock_bh(&sk->sk_callback_lock); + WRITE_ONCE(sk->sk_user_data, NULL); + RCU_INIT_POINTER(array->ptrs[index], NULL); + write_unlock_bh(&sk->sk_callback_lock); + err = 0; + } else { + err = -ENOENT; + } + + spin_unlock_bh(&reuseport_lock); + + return err; +} + +static void reuseport_array_free(struct bpf_map *map) +{ + struct reuseport_array *array = reuseport_array(map); + struct sock *sk; + u32 i; + + synchronize_rcu(); + + /* + * ops->map_*_elem() will not be able to access this + * array now. Hence, this function only races with + * bpf_sk_reuseport_detach() which was triggerred by + * close() or disconnect(). + * + * This function and bpf_sk_reuseport_detach() are + * both removing sk from "array". Who removes it + * first does not matter. + * + * The only concern here is bpf_sk_reuseport_detach() + * may access "array" which is being freed here. + * bpf_sk_reuseport_detach() access this "array" + * through sk->sk_user_data _and_ with sk->sk_callback_lock + * held which is enough because this "array" is not freed + * until all sk->sk_user_data has stopped referencing this "array". + * + * Hence, due to the above, taking "reuseport_lock" is not + * needed here. + */ + + /* + * Since reuseport_lock is not taken, sk is accessed under + * rcu_read_lock() + */ + rcu_read_lock(); + for (i = 0; i < map->max_entries; i++) { + sk = rcu_dereference(array->ptrs[i]); + if (sk) { + write_lock_bh(&sk->sk_callback_lock); + /* + * No need for WRITE_ONCE(). At this point, + * no one is reading it without taking the + * sk->sk_callback_lock. + */ + sk->sk_user_data = NULL; + write_unlock_bh(&sk->sk_callback_lock); + RCU_INIT_POINTER(array->ptrs[i], NULL); + } + } + rcu_read_unlock(); + + /* + * Once reaching here, all sk->sk_user_data is not + * referenceing this "array". "array" can be freed now. + */ + bpf_map_area_free(array); +} + +static struct bpf_map *reuseport_array_alloc(union bpf_attr *attr) +{ + int err, numa_node = bpf_map_attr_numa_node(attr); + struct reuseport_array *array; + u64 cost, array_size; + + if (!capable(CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + + array_size = sizeof(*array); + array_size += (u64)attr->max_entries * sizeof(struct sock *); + + /* make sure there is no u32 overflow later in round_up() */ + cost = array_size; + if (cost >= U32_MAX - PAGE_SIZE) + return ERR_PTR(-ENOMEM); + cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT; + + err = bpf_map_precharge_memlock(cost); + if (err) + return ERR_PTR(err); + + /* allocate all map elements and zero-initialize them */ + array = bpf_map_area_alloc(array_size, numa_node); + if (!array) + return ERR_PTR(-ENOMEM); + + /* copy mandatory map attributes */ + bpf_map_init_from_attr(&array->map, attr); + array->map.pages = cost; + + return &array->map; +} + +int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, + void *value) +{ + struct sock *sk; + int err; + + if (map->value_size != sizeof(u64)) + return -ENOSPC; + + rcu_read_lock(); + sk = reuseport_array_lookup_elem(map, key); + if (sk) { + *(u64 *)value = sock_gen_cookie(sk); + err = 0; + } else { + err = -ENOENT; + } + rcu_read_unlock(); + + return err; +} + +static int +reuseport_array_update_check(const struct reuseport_array *array, + const struct sock *nsk, + const struct sock *osk, + const struct sock_reuseport *nsk_reuse, + u32 map_flags) +{ + if (osk && map_flags == BPF_NOEXIST) + return -EEXIST; + + if (!osk && map_flags == BPF_EXIST) + return -ENOENT; + + if (nsk->sk_protocol != IPPROTO_UDP && nsk->sk_protocol != IPPROTO_TCP) + return -ENOTSUPP; + + if (nsk->sk_family != AF_INET && nsk->sk_family != AF_INET6) + return -ENOTSUPP; + + if (nsk->sk_type != SOCK_STREAM && nsk->sk_type != SOCK_DGRAM) + return -ENOTSUPP; + + /* + * sk must be hashed (i.e. listening in the TCP case or binded + * in the UDP case) and + * it must also be a SO_REUSEPORT sk (i.e. reuse cannot be NULL). + * + * Also, sk will be used in bpf helper that is protected by + * rcu_read_lock(). + */ + if (!sock_flag(nsk, SOCK_RCU_FREE) || !sk_hashed(nsk) || !nsk_reuse) + return -EINVAL; + + /* READ_ONCE because the sk->sk_callback_lock may not be held here */ + if (READ_ONCE(nsk->sk_user_data)) + return -EBUSY; + + return 0; +} + +/* + * Called from syscall only. + * The "nsk" in the fd refcnt. + * The "osk" and "reuse" are protected by reuseport_lock. + */ +int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, + void *value, u64 map_flags) +{ + struct reuseport_array *array = reuseport_array(map); + struct sock *free_osk = NULL, *osk, *nsk; + struct sock_reuseport *reuse; + u32 index = *(u32 *)key; + struct socket *socket; + int err, fd; + + if (map_flags > BPF_EXIST) + return -EINVAL; + + if (index >= map->max_entries) + return -E2BIG; + + if (map->value_size == sizeof(u64)) { + u64 fd64 = *(u64 *)value; + + if (fd64 > S32_MAX) + return -EINVAL; + fd = fd64; + } else { + fd = *(int *)value; + } + + socket = sockfd_lookup(fd, &err); + if (!socket) + return err; + + nsk = socket->sk; + if (!nsk) { + err = -EINVAL; + goto put_file; + } + + /* Quick checks before taking reuseport_lock */ + err = reuseport_array_update_check(array, nsk, + rcu_access_pointer(array->ptrs[index]), + rcu_access_pointer(nsk->sk_reuseport_cb), + map_flags); + if (err) + goto put_file; + + spin_lock_bh(&reuseport_lock); + /* + * Some of the checks only need reuseport_lock + * but it is done under sk_callback_lock also + * for simplicity reason. + */ + write_lock_bh(&nsk->sk_callback_lock); + + osk = rcu_dereference_protected(array->ptrs[index], + lockdep_is_held(&reuseport_lock)); + reuse = rcu_dereference_protected(nsk->sk_reuseport_cb, + lockdep_is_held(&reuseport_lock)); + err = reuseport_array_update_check(array, nsk, osk, reuse, map_flags); + if (err) + goto put_file_unlock; + + /* Ensure reuse->reuseport_id is set */ + err = reuseport_get_id(reuse); + if (err < 0) + goto put_file_unlock; + + WRITE_ONCE(nsk->sk_user_data, &array->ptrs[index]); + rcu_assign_pointer(array->ptrs[index], nsk); + free_osk = osk; + err = 0; + +put_file_unlock: + write_unlock_bh(&nsk->sk_callback_lock); + + if (free_osk) { + write_lock_bh(&free_osk->sk_callback_lock); + WRITE_ONCE(free_osk->sk_user_data, NULL); + write_unlock_bh(&free_osk->sk_callback_lock); + } + + spin_unlock_bh(&reuseport_lock); +put_file: + fput(socket->file); + return err; +} + +/* Called from syscall */ +static int reuseport_array_get_next_key(struct bpf_map *map, void *key, + void *next_key) +{ + struct reuseport_array *array = reuseport_array(map); + u32 index = key ? *(u32 *)key : U32_MAX; + u32 *next = (u32 *)next_key; + + if (index >= array->map.max_entries) { + *next = 0; + return 0; + } + + if (index == array->map.max_entries - 1) + return -ENOENT; + + *next = index + 1; + return 0; +} + +const struct bpf_map_ops reuseport_array_ops = { + .map_alloc_check = reuseport_array_alloc_check, + .map_alloc = reuseport_array_alloc, + .map_free = reuseport_array_free, + .map_lookup_elem = reuseport_array_lookup_elem, + .map_get_next_key = reuseport_array_get_next_key, + .map_delete_elem = reuseport_array_delete_elem, +}; diff --git a/kernel/bpf/sockmap.c b/kernel/bpf/sockmap.c index 0b38be5a955c..0c1a696b041b 100644 --- a/kernel/bpf/sockmap.c +++ b/kernel/bpf/sockmap.c @@ -1045,12 +1045,12 @@ static int bpf_tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); while (msg_data_left(msg)) { - struct sk_msg_buff *m; + struct sk_msg_buff *m = NULL; bool enospc = false; int copy; if (sk->sk_err) { - err = sk->sk_err; + err = -sk->sk_err; goto out_err; } @@ -1113,8 +1113,11 @@ wait_for_sndbuf: set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); wait_for_memory: err = sk_stream_wait_memory(sk, &timeo); - if (err) + if (err) { + if (m && m != psock->cork) + free_start_sg(sk, m); goto out_err; + } } out_err: if (err < 0) @@ -2495,6 +2498,7 @@ const struct bpf_map_ops sock_map_ops = { .map_update_elem = sock_map_update_elem, .map_delete_elem = sock_map_delete_elem, .map_release_uref = sock_map_release, + .map_check_btf = map_check_no_btf, }; const struct bpf_map_ops sock_hash_ops = { @@ -2505,6 +2509,7 @@ const struct bpf_map_ops sock_hash_ops = { .map_update_elem = sock_hash_update_elem, .map_delete_elem = sock_hash_delete_elem, .map_release_uref = sock_map_release, + .map_check_btf = map_check_no_btf, }; BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, bpf_sock, diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index b675a3f3d141..8061a439ef18 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -607,6 +607,7 @@ const struct bpf_map_ops stack_map_ops = { .map_lookup_elem = stack_map_lookup_elem, .map_update_elem = stack_map_update_elem, .map_delete_elem = stack_map_delete_elem, + .map_check_btf = map_check_no_btf, }; static int __init stack_map_init(void) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 5af4e9e2722d..43727ed0d94a 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -103,6 +103,7 @@ int bpf_check_uarg_tail_zero(void __user *uaddr, const struct bpf_map_ops bpf_map_offload_ops = { .map_alloc = bpf_map_offload_map_alloc, .map_free = bpf_map_offload_map_free, + .map_check_btf = map_check_no_btf, }; static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) @@ -455,6 +456,34 @@ static int bpf_obj_name_cpy(char *dst, const char *src) return 0; } +int map_check_no_btf(const struct bpf_map *map, + const struct btf_type *key_type, + const struct btf_type *value_type) +{ + return -ENOTSUPP; +} + +static int map_check_btf(const struct bpf_map *map, const struct btf *btf, + u32 btf_key_id, u32 btf_value_id) +{ + const struct btf_type *key_type, *value_type; + u32 key_size, value_size; + int ret = 0; + + key_type = btf_type_id_size(btf, &btf_key_id, &key_size); + if (!key_type || key_size != map->key_size) + return -EINVAL; + + value_type = btf_type_id_size(btf, &btf_value_id, &value_size); + if (!value_type || value_size != map->value_size) + return -EINVAL; + + if (map->ops->map_check_btf) + ret = map->ops->map_check_btf(map, key_type, value_type); + + return ret; +} + #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id /* called via syscall */ static int map_create(union bpf_attr *attr) @@ -489,8 +518,7 @@ static int map_create(union bpf_attr *attr) atomic_set(&map->refcnt, 1); atomic_set(&map->usercnt, 1); - if (bpf_map_support_seq_show(map) && - (attr->btf_key_type_id || attr->btf_value_type_id)) { + if (attr->btf_key_type_id || attr->btf_value_type_id) { struct btf *btf; if (!attr->btf_key_type_id || !attr->btf_value_type_id) { @@ -504,8 +532,8 @@ static int map_create(union bpf_attr *attr) goto free_map_nouncharge; } - err = map->ops->map_check_btf(map, btf, attr->btf_key_type_id, - attr->btf_value_type_id); + err = map_check_btf(map, btf, attr->btf_key_type_id, + attr->btf_value_type_id); if (err) { btf_put(btf); goto free_map_nouncharge; @@ -684,6 +712,8 @@ static int map_lookup_elem(union bpf_attr *attr) err = bpf_fd_array_map_lookup_elem(map, key, value); } else if (IS_FD_HASH(map)) { err = bpf_fd_htab_map_lookup_elem(map, key, value); + } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { + err = bpf_fd_reuseport_array_lookup_elem(map, key, value); } else { rcu_read_lock(); ptr = map->ops->map_lookup_elem(map, key); @@ -790,6 +820,10 @@ static int map_update_elem(union bpf_attr *attr) err = bpf_fd_htab_map_update_elem(map, f.file, key, value, attr->flags); rcu_read_unlock(); + } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { + /* rcu_read_lock() is not needed */ + err = bpf_fd_reuseport_array_update_elem(map, key, value, + attr->flags); } else { rcu_read_lock(); err = map->ops->map_update_elem(map, key, value, attr->flags); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 587468a9c37d..ca90679a7fe5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1310,6 +1310,7 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_IN: case BPF_PROG_TYPE_LWT_OUT: case BPF_PROG_TYPE_LWT_SEG6LOCAL: + case BPF_PROG_TYPE_SK_REUSEPORT: /* dst_input() and dst_output() can't write for now */ if (t == BPF_WRITE) return false; @@ -2166,6 +2167,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, func_id != BPF_FUNC_msg_redirect_hash) goto error; break; + case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: + if (func_id != BPF_FUNC_sk_select_reuseport) + goto error; + break; default: break; } @@ -2217,6 +2222,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE) goto error; break; + case BPF_FUNC_sk_select_reuseport: + if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) + goto error; + break; default: break; } diff --git a/kernel/bpf/xskmap.c b/kernel/bpf/xskmap.c index b3c557476a8d..4ddf61e158f6 100644 --- a/kernel/bpf/xskmap.c +++ b/kernel/bpf/xskmap.c @@ -227,6 +227,5 @@ const struct bpf_map_ops xsk_map_ops = { .map_lookup_elem = xsk_map_lookup_elem, .map_update_elem = xsk_map_update_elem, .map_delete_elem = xsk_map_delete_elem, + .map_check_btf = map_check_no_btf, }; - - diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 19d42ea75ec2..98fa559ebd80 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -1,9 +1,6 @@ config ARCH_HAS_UBSAN_SANITIZE_ALL bool -config ARCH_WANTS_UBSAN_NO_NULL - def_bool n - config UBSAN bool "Undefined behaviour sanity checker" help @@ -39,14 +36,6 @@ config UBSAN_ALIGNMENT Enabling this option on architectures that support unaligned accesses may produce a lot of false positives. -config UBSAN_NULL - bool "Enable checking of null pointers" - depends on UBSAN - default y if !ARCH_WANTS_UBSAN_NO_NULL - help - This option enables detection of memory accesses via a - null pointer. - config TEST_UBSAN tristate "Module for testing for undefined behavior detection" depends on m && UBSAN diff --git a/mm/memory.c b/mm/memory.c index dab1511294ad..c5e87a3a82ba 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4395,6 +4395,9 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, return -EINVAL; maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot); + if (!maddr) + return -ENOMEM; + if (write) memcpy_toio(maddr + offset, buf, len); else diff --git a/net/core/filter.c b/net/core/filter.c index 587bbfbd7db3..15b9d2df92ca 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1453,30 +1453,6 @@ static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk) return 0; } -static int __reuseport_attach_prog(struct bpf_prog *prog, struct sock *sk) -{ - struct bpf_prog *old_prog; - int err; - - if (bpf_prog_size(prog->len) > sysctl_optmem_max) - return -ENOMEM; - - if (sk_unhashed(sk) && sk->sk_reuseport) { - err = reuseport_alloc(sk); - if (err) - return err; - } else if (!rcu_access_pointer(sk->sk_reuseport_cb)) { - /* The socket wasn't bound with SO_REUSEPORT */ - return -EINVAL; - } - - old_prog = reuseport_attach_prog(sk, prog); - if (old_prog) - bpf_prog_destroy(old_prog); - - return 0; -} - static struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk) { @@ -1550,13 +1526,15 @@ int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk) if (IS_ERR(prog)) return PTR_ERR(prog); - err = __reuseport_attach_prog(prog, sk); - if (err < 0) { + if (bpf_prog_size(prog->len) > sysctl_optmem_max) + err = -ENOMEM; + else + err = reuseport_attach_prog(sk, prog); + + if (err) __bpf_prog_release(prog); - return err; - } - return 0; + return err; } static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk) @@ -1586,19 +1564,58 @@ int sk_attach_bpf(u32 ufd, struct sock *sk) int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk) { - struct bpf_prog *prog = __get_bpf(ufd, sk); + struct bpf_prog *prog; int err; + if (sock_flag(sk, SOCK_FILTER_LOCKED)) + return -EPERM; + + prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER); + if (IS_ERR(prog) && PTR_ERR(prog) == -EINVAL) + prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SK_REUSEPORT); if (IS_ERR(prog)) return PTR_ERR(prog); - err = __reuseport_attach_prog(prog, sk); - if (err < 0) { - bpf_prog_put(prog); - return err; + if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) { + /* Like other non BPF_PROG_TYPE_SOCKET_FILTER + * bpf prog (e.g. sockmap). It depends on the + * limitation imposed by bpf_prog_load(). + * Hence, sysctl_optmem_max is not checked. + */ + if ((sk->sk_type != SOCK_STREAM && + sk->sk_type != SOCK_DGRAM) || + (sk->sk_protocol != IPPROTO_UDP && + sk->sk_protocol != IPPROTO_TCP) || + (sk->sk_family != AF_INET && + sk->sk_family != AF_INET6)) { + err = -ENOTSUPP; + goto err_prog_put; + } + } else { + /* BPF_PROG_TYPE_SOCKET_FILTER */ + if (bpf_prog_size(prog->len) > sysctl_optmem_max) { + err = -ENOMEM; + goto err_prog_put; + } } - return 0; + err = reuseport_attach_prog(sk, prog); +err_prog_put: + if (err) + bpf_prog_put(prog); + + return err; +} + +void sk_reuseport_prog_free(struct bpf_prog *prog) +{ + if (!prog) + return; + + if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) + bpf_prog_put(prog); + else + bpf_prog_destroy(prog); } struct bpf_scratchpad { @@ -2082,19 +2099,12 @@ static const struct bpf_func_proto bpf_clone_redirect_proto = { .arg3_type = ARG_ANYTHING, }; -struct redirect_info { - u32 ifindex; - u32 flags; - struct bpf_map *map; - struct bpf_map *map_to_flush; - unsigned long map_owner; -}; - -static DEFINE_PER_CPU(struct redirect_info, redirect_info); +DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info); +EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info); BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); if (unlikely(flags & ~(BPF_F_INGRESS))) return TC_ACT_SHOT; @@ -2107,7 +2117,7 @@ BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) int skb_do_redirect(struct sk_buff *skb) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); struct net_device *dev; dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->ifindex); @@ -3200,7 +3210,7 @@ static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd, void xdp_do_flush_map(void) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); struct bpf_map *map = ri->map_to_flush; ri->map_to_flush = NULL; @@ -3245,7 +3255,7 @@ static inline bool xdp_map_invalid(const struct bpf_prog *xdp_prog, static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); unsigned long map_owner = ri->map_owner; struct bpf_map *map = ri->map; u32 index = ri->ifindex; @@ -3285,7 +3295,7 @@ err: int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); struct net_device *fwd; u32 index = ri->ifindex; int err; @@ -3317,7 +3327,7 @@ static int xdp_do_generic_redirect_map(struct net_device *dev, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); unsigned long map_owner = ri->map_owner; struct bpf_map *map = ri->map; u32 index = ri->ifindex; @@ -3368,7 +3378,7 @@ err: int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); u32 index = ri->ifindex; struct net_device *fwd; int err = 0; @@ -3399,7 +3409,7 @@ EXPORT_SYMBOL_GPL(xdp_do_generic_redirect); BPF_CALL_2(bpf_xdp_redirect, u32, ifindex, u64, flags) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); if (unlikely(flags)) return XDP_ABORTED; @@ -3423,7 +3433,7 @@ static const struct bpf_func_proto bpf_xdp_redirect_proto = { BPF_CALL_4(bpf_xdp_redirect_map, struct bpf_map *, map, u32, ifindex, u64, flags, unsigned long, map_owner) { - struct redirect_info *ri = this_cpu_ptr(&redirect_info); + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); if (unlikely(flags)) return XDP_ABORTED; @@ -3768,6 +3778,32 @@ static const struct bpf_func_proto bpf_skb_cgroup_id_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, }; + +BPF_CALL_2(bpf_skb_ancestor_cgroup_id, const struct sk_buff *, skb, int, + ancestor_level) +{ + struct sock *sk = skb_to_full_sk(skb); + struct cgroup *ancestor; + struct cgroup *cgrp; + + if (!sk || !sk_fullsock(sk)) + return 0; + + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); + ancestor = cgroup_ancestor(cgrp, ancestor_level); + if (!ancestor) + return 0; + + return ancestor->kn->id.id; +} + +static const struct bpf_func_proto bpf_skb_ancestor_cgroup_id_proto = { + .func = bpf_skb_ancestor_cgroup_id, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, +}; #endif static unsigned long bpf_xdp_copy(void *dst_buff, const void *src_buff, @@ -4956,6 +4992,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #ifdef CONFIG_SOCK_CGROUP_DATA case BPF_FUNC_skb_cgroup_id: return &bpf_skb_cgroup_id_proto; + case BPF_FUNC_skb_ancestor_cgroup_id: + return &bpf_skb_ancestor_cgroup_id_proto; #endif default: return bpf_base_func_proto(func_id); @@ -7020,3 +7058,270 @@ out: release_sock(sk); return ret; } + +#ifdef CONFIG_INET +struct sk_reuseport_kern { + struct sk_buff *skb; + struct sock *sk; + struct sock *selected_sk; + void *data_end; + u32 hash; + u32 reuseport_id; + bool bind_inany; +}; + +static void bpf_init_reuseport_kern(struct sk_reuseport_kern *reuse_kern, + struct sock_reuseport *reuse, + struct sock *sk, struct sk_buff *skb, + u32 hash) +{ + reuse_kern->skb = skb; + reuse_kern->sk = sk; + reuse_kern->selected_sk = NULL; + reuse_kern->data_end = skb->data + skb_headlen(skb); + reuse_kern->hash = hash; + reuse_kern->reuseport_id = reuse->reuseport_id; + reuse_kern->bind_inany = reuse->bind_inany; +} + +struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, + struct bpf_prog *prog, struct sk_buff *skb, + u32 hash) +{ + struct sk_reuseport_kern reuse_kern; + enum sk_action action; + + bpf_init_reuseport_kern(&reuse_kern, reuse, sk, skb, hash); + action = BPF_PROG_RUN(prog, &reuse_kern); + + if (action == SK_PASS) + return reuse_kern.selected_sk; + else + return ERR_PTR(-ECONNREFUSED); +} + +BPF_CALL_4(sk_select_reuseport, struct sk_reuseport_kern *, reuse_kern, + struct bpf_map *, map, void *, key, u32, flags) +{ + struct sock_reuseport *reuse; + struct sock *selected_sk; + + selected_sk = map->ops->map_lookup_elem(map, key); + if (!selected_sk) + return -ENOENT; + + reuse = rcu_dereference(selected_sk->sk_reuseport_cb); + if (!reuse) + /* selected_sk is unhashed (e.g. by close()) after the + * above map_lookup_elem(). Treat selected_sk has already + * been removed from the map. + */ + return -ENOENT; + + if (unlikely(reuse->reuseport_id != reuse_kern->reuseport_id)) { + struct sock *sk; + + if (unlikely(!reuse_kern->reuseport_id)) + /* There is a small race between adding the + * sk to the map and setting the + * reuse_kern->reuseport_id. + * Treat it as the sk has not been added to + * the bpf map yet. + */ + return -ENOENT; + + sk = reuse_kern->sk; + if (sk->sk_protocol != selected_sk->sk_protocol) + return -EPROTOTYPE; + else if (sk->sk_family != selected_sk->sk_family) + return -EAFNOSUPPORT; + + /* Catch all. Likely bound to a different sockaddr. */ + return -EBADFD; + } + + reuse_kern->selected_sk = selected_sk; + + return 0; +} + +static const struct bpf_func_proto sk_select_reuseport_proto = { + .func = sk_select_reuseport, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_CONST_MAP_PTR, + .arg3_type = ARG_PTR_TO_MAP_KEY, + .arg4_type = ARG_ANYTHING, +}; + +BPF_CALL_4(sk_reuseport_load_bytes, + const struct sk_reuseport_kern *, reuse_kern, u32, offset, + void *, to, u32, len) +{ + return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len); +} + +static const struct bpf_func_proto sk_reuseport_load_bytes_proto = { + .func = sk_reuseport_load_bytes, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_PTR_TO_UNINIT_MEM, + .arg4_type = ARG_CONST_SIZE, +}; + +BPF_CALL_5(sk_reuseport_load_bytes_relative, + const struct sk_reuseport_kern *, reuse_kern, u32, offset, + void *, to, u32, len, u32, start_header) +{ + return ____bpf_skb_load_bytes_relative(reuse_kern->skb, offset, to, + len, start_header); +} + +static const struct bpf_func_proto sk_reuseport_load_bytes_relative_proto = { + .func = sk_reuseport_load_bytes_relative, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_PTR_TO_UNINIT_MEM, + .arg4_type = ARG_CONST_SIZE, + .arg5_type = ARG_ANYTHING, +}; + +static const struct bpf_func_proto * +sk_reuseport_func_proto(enum bpf_func_id func_id, + const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_sk_select_reuseport: + return &sk_select_reuseport_proto; + case BPF_FUNC_skb_load_bytes: + return &sk_reuseport_load_bytes_proto; + case BPF_FUNC_skb_load_bytes_relative: + return &sk_reuseport_load_bytes_relative_proto; + default: + return bpf_base_func_proto(func_id); + } +} + +static bool +sk_reuseport_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + const u32 size_default = sizeof(__u32); + + if (off < 0 || off >= sizeof(struct sk_reuseport_md) || + off % size || type != BPF_READ) + return false; + + switch (off) { + case offsetof(struct sk_reuseport_md, data): + info->reg_type = PTR_TO_PACKET; + return size == sizeof(__u64); + + case offsetof(struct sk_reuseport_md, data_end): + info->reg_type = PTR_TO_PACKET_END; + return size == sizeof(__u64); + + case offsetof(struct sk_reuseport_md, hash): + return size == size_default; + + /* Fields that allow narrowing */ + case offsetof(struct sk_reuseport_md, eth_protocol): + if (size < FIELD_SIZEOF(struct sk_buff, protocol)) + return false; + case offsetof(struct sk_reuseport_md, ip_protocol): + case offsetof(struct sk_reuseport_md, bind_inany): + case offsetof(struct sk_reuseport_md, len): + bpf_ctx_record_field_size(info, size_default); + return bpf_ctx_narrow_access_ok(off, size, size_default); + + default: + return false; + } +} + +#define SK_REUSEPORT_LOAD_FIELD(F) ({ \ + *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_reuseport_kern, F), \ + si->dst_reg, si->src_reg, \ + bpf_target_off(struct sk_reuseport_kern, F, \ + FIELD_SIZEOF(struct sk_reuseport_kern, F), \ + target_size)); \ + }) + +#define SK_REUSEPORT_LOAD_SKB_FIELD(SKB_FIELD) \ + SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern, \ + struct sk_buff, \ + skb, \ + SKB_FIELD) + +#define SK_REUSEPORT_LOAD_SK_FIELD_SIZE_OFF(SK_FIELD, BPF_SIZE, EXTRA_OFF) \ + SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(struct sk_reuseport_kern, \ + struct sock, \ + sk, \ + SK_FIELD, BPF_SIZE, EXTRA_OFF) + +static u32 sk_reuseport_convert_ctx_access(enum bpf_access_type type, + const struct bpf_insn *si, + struct bpf_insn *insn_buf, + struct bpf_prog *prog, + u32 *target_size) +{ + struct bpf_insn *insn = insn_buf; + + switch (si->off) { + case offsetof(struct sk_reuseport_md, data): + SK_REUSEPORT_LOAD_SKB_FIELD(data); + break; + + case offsetof(struct sk_reuseport_md, len): + SK_REUSEPORT_LOAD_SKB_FIELD(len); + break; + + case offsetof(struct sk_reuseport_md, eth_protocol): + SK_REUSEPORT_LOAD_SKB_FIELD(protocol); + break; + + case offsetof(struct sk_reuseport_md, ip_protocol): + BUILD_BUG_ON(hweight_long(SK_FL_PROTO_MASK) != BITS_PER_BYTE); + SK_REUSEPORT_LOAD_SK_FIELD_SIZE_OFF(__sk_flags_offset, + BPF_W, 0); + *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, SK_FL_PROTO_MASK); + *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, + SK_FL_PROTO_SHIFT); + /* SK_FL_PROTO_MASK and SK_FL_PROTO_SHIFT are endian + * aware. No further narrowing or masking is needed. + */ + *target_size = 1; + break; + + case offsetof(struct sk_reuseport_md, data_end): + SK_REUSEPORT_LOAD_FIELD(data_end); + break; + + case offsetof(struct sk_reuseport_md, hash): + SK_REUSEPORT_LOAD_FIELD(hash); + break; + + case offsetof(struct sk_reuseport_md, bind_inany): + SK_REUSEPORT_LOAD_FIELD(bind_inany); + break; + } + + return insn - insn_buf; +} + +const struct bpf_verifier_ops sk_reuseport_verifier_ops = { + .get_func_proto = sk_reuseport_func_proto, + .is_valid_access = sk_reuseport_is_valid_access, + .convert_ctx_access = sk_reuseport_convert_ctx_access, +}; + +const struct bpf_prog_ops sk_reuseport_prog_ops = { +}; +#endif /* CONFIG_INET */ diff --git a/net/core/gen_estimator.c b/net/core/gen_estimator.c index 98fd12721221..e4e442d70c2d 100644 --- a/net/core/gen_estimator.c +++ b/net/core/gen_estimator.c @@ -112,7 +112,7 @@ static void est_timer(struct timer_list *t) * @bstats: basic statistics * @cpu_bstats: bstats per cpu * @rate_est: rate estimator statistics - * @stats_lock: statistics lock + * @lock: lock for statistics and control path * @running: qdisc running seqcount * @opt: rate estimator configuration TLV * @@ -128,7 +128,7 @@ static void est_timer(struct timer_list *t) int gen_new_estimator(struct gnet_stats_basic_packed *bstats, struct gnet_stats_basic_cpu __percpu *cpu_bstats, struct net_rate_estimator __rcu **rate_est, - spinlock_t *stats_lock, + spinlock_t *lock, seqcount_t *running, struct nlattr *opt) { @@ -154,19 +154,22 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats, seqcount_init(&est->seq); intvl_log = parm->interval + 2; est->bstats = bstats; - est->stats_lock = stats_lock; + est->stats_lock = lock; est->running = running; est->ewma_log = parm->ewma_log; est->intvl_log = intvl_log; est->cpu_bstats = cpu_bstats; - if (stats_lock) + if (lock) local_bh_disable(); est_fetch_counters(est, &b); - if (stats_lock) + if (lock) local_bh_enable(); est->last_bytes = b.bytes; est->last_packets = b.packets; + + if (lock) + spin_lock_bh(lock); old = rcu_dereference_protected(*rate_est, 1); if (old) { del_timer_sync(&old->timer); @@ -179,6 +182,8 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats, mod_timer(&est->timer, est->next_jiffies); rcu_assign_pointer(*rate_est, est); + if (lock) + spin_unlock_bh(lock); if (old) kfree_rcu(old, rcu); return 0; @@ -209,7 +214,7 @@ EXPORT_SYMBOL(gen_kill_estimator); * @bstats: basic statistics * @cpu_bstats: bstats per cpu * @rate_est: rate estimator statistics - * @stats_lock: statistics lock + * @lock: lock for statistics and control path * @running: qdisc running seqcount (might be NULL) * @opt: rate estimator configuration TLV * @@ -221,11 +226,11 @@ EXPORT_SYMBOL(gen_kill_estimator); int gen_replace_estimator(struct gnet_stats_basic_packed *bstats, struct gnet_stats_basic_cpu __percpu *cpu_bstats, struct net_rate_estimator __rcu **rate_est, - spinlock_t *stats_lock, + spinlock_t *lock, seqcount_t *running, struct nlattr *opt) { return gen_new_estimator(bstats, cpu_bstats, rate_est, - stats_lock, running, opt); + lock, running, opt); } EXPORT_SYMBOL(gen_replace_estimator); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 8d574a88125d..c996c09d095f 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1291,7 +1291,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) } EXPORT_SYMBOL(skb_clone); -static void skb_headers_offset_update(struct sk_buff *skb, int off) +void skb_headers_offset_update(struct sk_buff *skb, int off) { /* Only adjust this if it actually is csum_start rather than csum */ if (skb->ip_summed == CHECKSUM_PARTIAL) @@ -1305,6 +1305,7 @@ static void skb_headers_offset_update(struct sk_buff *skb, int off) skb->inner_network_header += off; skb->inner_mac_header += off; } +EXPORT_SYMBOL(skb_headers_offset_update); void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) { diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c index 064acb04be0f..ba5cba56f574 100644 --- a/net/core/sock_reuseport.c +++ b/net/core/sock_reuseport.c @@ -8,11 +8,34 @@ #include <net/sock_reuseport.h> #include <linux/bpf.h> +#include <linux/idr.h> +#include <linux/filter.h> #include <linux/rcupdate.h> #define INIT_SOCKS 128 -static DEFINE_SPINLOCK(reuseport_lock); +DEFINE_SPINLOCK(reuseport_lock); + +#define REUSEPORT_MIN_ID 1 +static DEFINE_IDA(reuseport_ida); + +int reuseport_get_id(struct sock_reuseport *reuse) +{ + int id; + + if (reuse->reuseport_id) + return reuse->reuseport_id; + + id = ida_simple_get(&reuseport_ida, REUSEPORT_MIN_ID, 0, + /* Called under reuseport_lock */ + GFP_ATOMIC); + if (id < 0) + return id; + + reuse->reuseport_id = id; + + return reuse->reuseport_id; +} static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks) { @@ -29,7 +52,7 @@ static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks) return reuse; } -int reuseport_alloc(struct sock *sk) +int reuseport_alloc(struct sock *sk, bool bind_inany) { struct sock_reuseport *reuse; @@ -41,9 +64,17 @@ int reuseport_alloc(struct sock *sk) /* Allocation attempts can occur concurrently via the setsockopt path * and the bind/hash path. Nothing to do when we lose the race. */ - if (rcu_dereference_protected(sk->sk_reuseport_cb, - lockdep_is_held(&reuseport_lock))) + reuse = rcu_dereference_protected(sk->sk_reuseport_cb, + lockdep_is_held(&reuseport_lock)); + if (reuse) { + /* Only set reuse->bind_inany if the bind_inany is true. + * Otherwise, it will overwrite the reuse->bind_inany + * which was set by the bind/hash path. + */ + if (bind_inany) + reuse->bind_inany = bind_inany; goto out; + } reuse = __reuseport_alloc(INIT_SOCKS); if (!reuse) { @@ -53,6 +84,7 @@ int reuseport_alloc(struct sock *sk) reuse->socks[0] = sk; reuse->num_socks = 1; + reuse->bind_inany = bind_inany; rcu_assign_pointer(sk->sk_reuseport_cb, reuse); out: @@ -78,9 +110,12 @@ static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse) more_reuse->max_socks = more_socks_size; more_reuse->num_socks = reuse->num_socks; more_reuse->prog = reuse->prog; + more_reuse->reuseport_id = reuse->reuseport_id; + more_reuse->bind_inany = reuse->bind_inany; memcpy(more_reuse->socks, reuse->socks, reuse->num_socks * sizeof(struct sock *)); + more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts); for (i = 0; i < reuse->num_socks; ++i) rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb, @@ -99,8 +134,9 @@ static void reuseport_free_rcu(struct rcu_head *head) struct sock_reuseport *reuse; reuse = container_of(head, struct sock_reuseport, rcu); - if (reuse->prog) - bpf_prog_destroy(reuse->prog); + sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1)); + if (reuse->reuseport_id) + ida_simple_remove(&reuseport_ida, reuse->reuseport_id); kfree(reuse); } @@ -110,12 +146,12 @@ static void reuseport_free_rcu(struct rcu_head *head) * @sk2: Socket belonging to the existing reuseport group. * May return ENOMEM and not add socket to group under memory pressure. */ -int reuseport_add_sock(struct sock *sk, struct sock *sk2) +int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany) { struct sock_reuseport *old_reuse, *reuse; if (!rcu_access_pointer(sk2->sk_reuseport_cb)) { - int err = reuseport_alloc(sk2); + int err = reuseport_alloc(sk2, bind_inany); if (err) return err; @@ -160,6 +196,14 @@ void reuseport_detach_sock(struct sock *sk) spin_lock_bh(&reuseport_lock); reuse = rcu_dereference_protected(sk->sk_reuseport_cb, lockdep_is_held(&reuseport_lock)); + + /* At least one of the sk in this reuseport group is added to + * a bpf map. Notify the bpf side. The bpf map logic will + * remove the sk if it is indeed added to a bpf map. + */ + if (reuse->reuseport_id) + bpf_sk_reuseport_detach(sk); + rcu_assign_pointer(sk->sk_reuseport_cb, NULL); for (i = 0; i < reuse->num_socks; i++) { @@ -175,9 +219,9 @@ void reuseport_detach_sock(struct sock *sk) } EXPORT_SYMBOL(reuseport_detach_sock); -static struct sock *run_bpf(struct sock_reuseport *reuse, u16 socks, - struct bpf_prog *prog, struct sk_buff *skb, - int hdr_len) +static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks, + struct bpf_prog *prog, struct sk_buff *skb, + int hdr_len) { struct sk_buff *nskb = NULL; u32 index; @@ -238,9 +282,15 @@ struct sock *reuseport_select_sock(struct sock *sk, /* paired with smp_wmb() in reuseport_add_sock() */ smp_rmb(); - if (prog && skb) - sk2 = run_bpf(reuse, socks, prog, skb, hdr_len); + if (!prog || !skb) + goto select_by_hash; + + if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) + sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, hash); + else + sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len); +select_by_hash: /* no bpf or invalid bpf result: fall back to hash usage */ if (!sk2) sk2 = reuse->socks[reciprocal_scale(hash, socks)]; @@ -252,12 +302,21 @@ out: } EXPORT_SYMBOL(reuseport_select_sock); -struct bpf_prog * -reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog) +int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog) { struct sock_reuseport *reuse; struct bpf_prog *old_prog; + if (sk_unhashed(sk) && sk->sk_reuseport) { + int err = reuseport_alloc(sk, false); + + if (err) + return err; + } else if (!rcu_access_pointer(sk->sk_reuseport_cb)) { + /* The socket wasn't bound with SO_REUSEPORT */ + return -EINVAL; + } + spin_lock_bh(&reuseport_lock); reuse = rcu_dereference_protected(sk->sk_reuseport_cb, lockdep_is_held(&reuseport_lock)); @@ -266,6 +325,7 @@ reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog) rcu_assign_pointer(reuse->prog, prog); spin_unlock_bh(&reuseport_lock); - return old_prog; + sk_reuseport_prog_free(old_prog); + return 0; } EXPORT_SYMBOL(reuseport_attach_prog); diff --git a/net/core/xdp.c b/net/core/xdp.c index c013b836006b..3dd99e1c04f5 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -330,10 +330,12 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, /* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */ xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); page = virt_to_head_page(data); - if (xa) + if (xa) { + napi_direct &= !xdp_return_frame_no_direct(); page_pool_put_page(xa->page_pool, page, napi_direct); - else + } else { put_page(page); + } rcu_read_unlock(); break; case MEM_TYPE_PAGE_SHARED: @@ -348,8 +350,7 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, rcu_read_lock(); /* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */ xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); - if (!WARN_ON_ONCE(!xa)) - xa->zc_alloc->free(xa->zc_alloc, handle); + xa->zc_alloc->free(xa->zc_alloc, handle); rcu_read_unlock(); default: /* Not possible, checked in xdp_rxq_info_reg_mem_model() */ diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 33a88e045efd..dfd5009f96ef 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -107,6 +107,15 @@ bool inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2, } EXPORT_SYMBOL(inet_rcv_saddr_equal); +bool inet_rcv_saddr_any(const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) + return ipv6_addr_any(&sk->sk_v6_rcv_saddr); +#endif + return !sk->sk_rcv_saddr; +} + void inet_get_local_port_range(struct net *net, int *low, int *high) { unsigned int seq; diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c index 6d258a5669e7..bcb11f3a27c0 100644 --- a/net/ipv4/inet_fragment.c +++ b/net/ipv4/inet_fragment.c @@ -146,7 +146,7 @@ void inet_frag_destroy(struct inet_frag_queue *q) fp = xp; } while (fp); } else { - sum_truesize = skb_rbtree_purge(&q->rb_fragments); + sum_truesize = inet_frag_rbtree_purge(&q->rb_fragments); } sum = sum_truesize + f->qsize; diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index 3647167c8fa3..f5c9ef2586de 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -328,7 +328,7 @@ struct sock *__inet_lookup_listener(struct net *net, saddr, sport, daddr, hnum, dif, sdif); if (result) - return result; + goto done; /* Lookup lhash2 with INADDR_ANY */ @@ -337,9 +337,10 @@ struct sock *__inet_lookup_listener(struct net *net, if (ilb2->count > ilb->count) goto port_lookup; - return inet_lhash2_lookup(net, ilb2, skb, doff, - saddr, sport, daddr, hnum, - dif, sdif); + result = inet_lhash2_lookup(net, ilb2, skb, doff, + saddr, sport, daddr, hnum, + dif, sdif); + goto done; port_lookup: sk_for_each_rcu(sk, &ilb->head) { @@ -352,12 +353,15 @@ port_lookup: result = reuseport_select_sock(sk, phash, skb, doff); if (result) - return result; + goto done; } result = sk; hiscore = score; } } +done: + if (unlikely(IS_ERR(result))) + return NULL; return result; } EXPORT_SYMBOL_GPL(__inet_lookup_listener); @@ -567,10 +571,11 @@ static int inet_reuseport_add_sock(struct sock *sk, inet_csk(sk2)->icsk_bind_hash == tb && sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && inet_rcv_saddr_equal(sk, sk2, false)) - return reuseport_add_sock(sk, sk2); + return reuseport_add_sock(sk, sk2, + inet_rcv_saddr_any(sk)); } - return reuseport_alloc(sk); + return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); } int __inet_hash(struct sock *sk, struct sock *osk) diff --git a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c index 7cb7ed761d8c..88281fbce88c 100644 --- a/net/ipv4/ip_fragment.c +++ b/net/ipv4/ip_fragment.c @@ -57,6 +57,57 @@ */ static const char ip_frag_cache_name[] = "ip4-frags"; +/* Use skb->cb to track consecutive/adjacent fragments coming at + * the end of the queue. Nodes in the rb-tree queue will + * contain "runs" of one or more adjacent fragments. + * + * Invariants: + * - next_frag is NULL at the tail of a "run"; + * - the head of a "run" has the sum of all fragment lengths in frag_run_len. + */ +struct ipfrag_skb_cb { + struct inet_skb_parm h; + struct sk_buff *next_frag; + int frag_run_len; +}; + +#define FRAG_CB(skb) ((struct ipfrag_skb_cb *)((skb)->cb)) + +static void ip4_frag_init_run(struct sk_buff *skb) +{ + BUILD_BUG_ON(sizeof(struct ipfrag_skb_cb) > sizeof(skb->cb)); + + FRAG_CB(skb)->next_frag = NULL; + FRAG_CB(skb)->frag_run_len = skb->len; +} + +/* Append skb to the last "run". */ +static void ip4_frag_append_to_last_run(struct inet_frag_queue *q, + struct sk_buff *skb) +{ + RB_CLEAR_NODE(&skb->rbnode); + FRAG_CB(skb)->next_frag = NULL; + + FRAG_CB(q->last_run_head)->frag_run_len += skb->len; + FRAG_CB(q->fragments_tail)->next_frag = skb; + q->fragments_tail = skb; +} + +/* Create a new "run" with the skb. */ +static void ip4_frag_create_run(struct inet_frag_queue *q, struct sk_buff *skb) +{ + if (q->last_run_head) + rb_link_node(&skb->rbnode, &q->last_run_head->rbnode, + &q->last_run_head->rbnode.rb_right); + else + rb_link_node(&skb->rbnode, NULL, &q->rb_fragments.rb_node); + rb_insert_color(&skb->rbnode, &q->rb_fragments); + + ip4_frag_init_run(skb); + q->fragments_tail = skb; + q->last_run_head = skb; +} + /* Describe an entry in the "incomplete datagrams" queue. */ struct ipq { struct inet_frag_queue q; @@ -75,8 +126,8 @@ static u8 ip4_frag_ecn(u8 tos) static struct inet_frags ip4_frags; -static int ip_frag_reasm(struct ipq *qp, struct sk_buff *prev, - struct net_device *dev); +static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, + struct sk_buff *prev_tail, struct net_device *dev); static void ip4_frag_init(struct inet_frag_queue *q, const void *a) @@ -168,7 +219,12 @@ static void ip_expire(struct timer_list *t) head = skb_rb_first(&qp->q.rb_fragments); if (!head) goto out; - rb_erase(&head->rbnode, &qp->q.rb_fragments); + if (FRAG_CB(head)->next_frag) + rb_replace_node(&head->rbnode, + &FRAG_CB(head)->next_frag->rbnode, + &qp->q.rb_fragments); + else + rb_erase(&head->rbnode, &qp->q.rb_fragments); memset(&head->rbnode, 0, sizeof(head->rbnode)); barrier(); } @@ -269,7 +325,7 @@ static int ip_frag_reinit(struct ipq *qp) return -ETIMEDOUT; } - sum_truesize = skb_rbtree_purge(&qp->q.rb_fragments); + sum_truesize = inet_frag_rbtree_purge(&qp->q.rb_fragments); sub_frag_mem_limit(qp->q.net, sum_truesize); qp->q.flags = 0; @@ -278,6 +334,7 @@ static int ip_frag_reinit(struct ipq *qp) qp->q.fragments = NULL; qp->q.rb_fragments = RB_ROOT; qp->q.fragments_tail = NULL; + qp->q.last_run_head = NULL; qp->iif = 0; qp->ecn = 0; @@ -289,7 +346,7 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) { struct net *net = container_of(qp->q.net, struct net, ipv4.frags); struct rb_node **rbn, *parent; - struct sk_buff *skb1; + struct sk_buff *skb1, *prev_tail; struct net_device *dev; unsigned int fragsize; int flags, offset; @@ -367,38 +424,41 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) */ /* Find out where to put this fragment. */ - skb1 = qp->q.fragments_tail; - if (!skb1) { - /* This is the first fragment we've received. */ - rb_link_node(&skb->rbnode, NULL, &qp->q.rb_fragments.rb_node); - qp->q.fragments_tail = skb; - } else if ((skb1->ip_defrag_offset + skb1->len) < end) { - /* This is the common/special case: skb goes to the end. */ + prev_tail = qp->q.fragments_tail; + if (!prev_tail) + ip4_frag_create_run(&qp->q, skb); /* First fragment. */ + else if (prev_tail->ip_defrag_offset + prev_tail->len < end) { + /* This is the common case: skb goes to the end. */ /* Detect and discard overlaps. */ - if (offset < (skb1->ip_defrag_offset + skb1->len)) + if (offset < prev_tail->ip_defrag_offset + prev_tail->len) goto discard_qp; - /* Insert after skb1. */ - rb_link_node(&skb->rbnode, &skb1->rbnode, &skb1->rbnode.rb_right); - qp->q.fragments_tail = skb; + if (offset == prev_tail->ip_defrag_offset + prev_tail->len) + ip4_frag_append_to_last_run(&qp->q, skb); + else + ip4_frag_create_run(&qp->q, skb); } else { - /* Binary search. Note that skb can become the first fragment, but - * not the last (covered above). */ + /* Binary search. Note that skb can become the first fragment, + * but not the last (covered above). + */ rbn = &qp->q.rb_fragments.rb_node; do { parent = *rbn; skb1 = rb_to_skb(parent); if (end <= skb1->ip_defrag_offset) rbn = &parent->rb_left; - else if (offset >= skb1->ip_defrag_offset + skb1->len) + else if (offset >= skb1->ip_defrag_offset + + FRAG_CB(skb1)->frag_run_len) rbn = &parent->rb_right; else /* Found an overlap with skb1. */ goto discard_qp; } while (*rbn); /* Here we have parent properly set, and rbn pointing to - * one of its NULL left/right children. Insert skb. */ + * one of its NULL left/right children. Insert skb. + */ + ip4_frag_init_run(skb); rb_link_node(&skb->rbnode, parent, rbn); + rb_insert_color(&skb->rbnode, &qp->q.rb_fragments); } - rb_insert_color(&skb->rbnode, &qp->q.rb_fragments); if (dev) qp->iif = dev->ifindex; @@ -425,7 +485,7 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) unsigned long orefdst = skb->_skb_refdst; skb->_skb_refdst = 0UL; - err = ip_frag_reasm(qp, skb, dev); + err = ip_frag_reasm(qp, skb, prev_tail, dev); skb->_skb_refdst = orefdst; return err; } @@ -444,7 +504,7 @@ err: /* Build a new IP datagram from all its fragments. */ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, - struct net_device *dev) + struct sk_buff *prev_tail, struct net_device *dev) { struct net *net = container_of(qp->q.net, struct net, ipv4.frags); struct iphdr *iph; @@ -468,10 +528,16 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, fp = skb_clone(skb, GFP_ATOMIC); if (!fp) goto out_nomem; - rb_replace_node(&skb->rbnode, &fp->rbnode, &qp->q.rb_fragments); + FRAG_CB(fp)->next_frag = FRAG_CB(skb)->next_frag; + if (RB_EMPTY_NODE(&skb->rbnode)) + FRAG_CB(prev_tail)->next_frag = fp; + else + rb_replace_node(&skb->rbnode, &fp->rbnode, + &qp->q.rb_fragments); if (qp->q.fragments_tail == skb) qp->q.fragments_tail = fp; skb_morph(skb, head); + FRAG_CB(skb)->next_frag = FRAG_CB(head)->next_frag; rb_replace_node(&head->rbnode, &skb->rbnode, &qp->q.rb_fragments); consume_skb(head); @@ -507,7 +573,7 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, for (i = 0; i < skb_shinfo(head)->nr_frags; i++) plen += skb_frag_size(&skb_shinfo(head)->frags[i]); clone->len = clone->data_len = head->data_len - plen; - skb->truesize += clone->truesize; + head->truesize += clone->truesize; clone->csum = 0; clone->ip_summed = head->ip_summed; add_frag_mem_limit(qp->q.net, clone->truesize); @@ -520,24 +586,36 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, skb_push(head, head->data - skb_network_header(head)); /* Traverse the tree in order, to build frag_list. */ + fp = FRAG_CB(head)->next_frag; rbn = rb_next(&head->rbnode); rb_erase(&head->rbnode, &qp->q.rb_fragments); - while (rbn) { - struct rb_node *rbnext = rb_next(rbn); - fp = rb_to_skb(rbn); - rb_erase(rbn, &qp->q.rb_fragments); - rbn = rbnext; - *nextp = fp; - nextp = &fp->next; - fp->prev = NULL; - memset(&fp->rbnode, 0, sizeof(fp->rbnode)); - head->data_len += fp->len; - head->len += fp->len; - if (head->ip_summed != fp->ip_summed) - head->ip_summed = CHECKSUM_NONE; - else if (head->ip_summed == CHECKSUM_COMPLETE) - head->csum = csum_add(head->csum, fp->csum); - head->truesize += fp->truesize; + while (rbn || fp) { + /* fp points to the next sk_buff in the current run; + * rbn points to the next run. + */ + /* Go through the current run. */ + while (fp) { + *nextp = fp; + nextp = &fp->next; + fp->prev = NULL; + memset(&fp->rbnode, 0, sizeof(fp->rbnode)); + head->data_len += fp->len; + head->len += fp->len; + if (head->ip_summed != fp->ip_summed) + head->ip_summed = CHECKSUM_NONE; + else if (head->ip_summed == CHECKSUM_COMPLETE) + head->csum = csum_add(head->csum, fp->csum); + head->truesize += fp->truesize; + fp = FRAG_CB(fp)->next_frag; + } + /* Move to the next run. */ + if (rbn) { + struct rb_node *rbnext = rb_next(rbn); + + fp = rb_to_skb(rbn); + rb_erase(rbn, &qp->q.rb_fragments); + rbn = rbnext; + } } sub_frag_mem_limit(qp->q.net, head->truesize); @@ -573,6 +651,7 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, qp->q.fragments = NULL; qp->q.rb_fragments = RB_ROOT; qp->q.fragments_tail = NULL; + qp->q.last_run_head = NULL; return 0; out_nomem: @@ -654,6 +733,28 @@ struct sk_buff *ip_check_defrag(struct net *net, struct sk_buff *skb, u32 user) } EXPORT_SYMBOL(ip_check_defrag); +unsigned int inet_frag_rbtree_purge(struct rb_root *root) +{ + struct rb_node *p = rb_first(root); + unsigned int sum = 0; + + while (p) { + struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); + + p = rb_next(p); + rb_erase(&skb->rbnode, root); + while (skb) { + struct sk_buff *next = FRAG_CB(skb)->next_frag; + + sum += skb->truesize; + kfree_skb(skb); + skb = next; + } + } + return sum; +} +EXPORT_SYMBOL(inet_frag_rbtree_purge); + #ifdef CONFIG_SYSCTL static int dist_min; diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c index 8b637f9f23a2..ca61e2a659e7 100644 --- a/net/ipv4/tcp_dctcp.c +++ b/net/ipv4/tcp_dctcp.c @@ -136,7 +136,7 @@ static void dctcp_ce_state_0_to_1(struct sock *sk) */ if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) __tcp_send_ack(sk, ca->prior_rcv_nxt); - tcp_enter_quickack_mode(sk, 1); + inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW; } ca->prior_rcv_nxt = tp->rcv_nxt; @@ -157,7 +157,7 @@ static void dctcp_ce_state_1_to_0(struct sock *sk) */ if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) __tcp_send_ack(sk, ca->prior_rcv_nxt); - tcp_enter_quickack_mode(sk, 1); + inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW; } ca->prior_rcv_nxt = tp->rcv_nxt; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 715d541b52dd..4c2dd9f863f7 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -245,16 +245,16 @@ static void tcp_ecn_queue_cwr(struct tcp_sock *tp) tp->ecn_flags |= TCP_ECN_QUEUE_CWR; } -static void tcp_ecn_accept_cwr(struct tcp_sock *tp, const struct sk_buff *skb) +static void tcp_ecn_accept_cwr(struct sock *sk, const struct sk_buff *skb) { if (tcp_hdr(skb)->cwr) { - tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR; + tcp_sk(sk)->ecn_flags &= ~TCP_ECN_DEMAND_CWR; /* If the sender is telling us it has entered CWR, then its * cwnd may be very low (even just 1 packet), so we should ACK * immediately. */ - tcp_enter_quickack_mode((struct sock *)tp, 2); + inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW; } } @@ -4703,7 +4703,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) skb_dst_drop(skb); __skb_pull(skb, tcp_hdr(skb)->doff * 4); - tcp_ecn_accept_cwr(tp, skb); + tcp_ecn_accept_cwr(sk, skb); tp->rx_opt.dsack = 0; @@ -4735,11 +4735,11 @@ queue_and_out: if (!RB_EMPTY_ROOT(&tp->out_of_order_queue)) { tcp_ofo_queue(sk); - /* RFC2581. 4.2. SHOULD send immediate ACK, when + /* RFC5681. 4.2. SHOULD send immediate ACK, when * gap in queue is filled. */ if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) - inet_csk(sk)->icsk_ack.pingpong = 0; + inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW; } if (tp->rx_opt.num_sacks) @@ -5179,7 +5179,9 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || __tcp_select_window(sk) >= tp->rcv_wnd)) || /* We ACK each frame or... */ - tcp_in_quickack_mode(sk)) { + tcp_in_quickack_mode(sk) || + /* Protocol state mandates a one-time immediate ACK */ + inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOW) { send_now: tcp_send_ack(sk); return; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 060e841dde40..f4e35b2ff8b8 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -221,11 +221,12 @@ static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot) (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && inet_rcv_saddr_equal(sk, sk2, false)) { - return reuseport_add_sock(sk, sk2); + return reuseport_add_sock(sk, sk2, + inet_rcv_saddr_any(sk)); } } - return reuseport_alloc(sk); + return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); } /** @@ -498,6 +499,8 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, daddr, hnum, dif, sdif, exact_dif, hslot2, skb); } + if (unlikely(IS_ERR(result))) + return NULL; return result; } begin: @@ -512,6 +515,8 @@ begin: saddr, sport); result = reuseport_select_sock(sk, hash, skb, sizeof(struct udphdr)); + if (unlikely(IS_ERR(result))) + return NULL; if (result) return result; } diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 020f6e14a7af..673bba31eb18 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -832,6 +832,7 @@ static int __net_init inet6_net_init(struct net *net) net->ipv6.sysctl.bindv6only = 0; net->ipv6.sysctl.icmpv6_time = 1*HZ; + net->ipv6.sysctl.icmpv6_echo_ignore_all = 0; net->ipv6.sysctl.flowlabel_consistency = 1; net->ipv6.sysctl.auto_flowlabels = IP6_DEFAULT_AUTO_FLOW_LABELS; net->ipv6.sysctl.idgen_retries = 3; diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index 7f6b1f81c200..c9c53ade55c3 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -794,6 +794,7 @@ out: static int icmpv6_rcv(struct sk_buff *skb) { + struct net *net = dev_net(skb->dev); struct net_device *dev = skb->dev; struct inet6_dev *idev = __in6_dev_get(dev); const struct in6_addr *saddr, *daddr; @@ -843,7 +844,8 @@ static int icmpv6_rcv(struct sk_buff *skb) switch (type) { case ICMPV6_ECHO_REQUEST: - icmpv6_echo_reply(skb); + if (!net->ipv6.sysctl.icmpv6_echo_ignore_all) + icmpv6_echo_reply(skb); break; case ICMPV6_ECHO_REPLY: @@ -1104,6 +1106,13 @@ static struct ctl_table ipv6_icmp_table_template[] = { .mode = 0644, .proc_handler = proc_dointvec_ms_jiffies, }, + { + .procname = "echo_ignore_all", + .data = &init_net.ipv6.sysctl.icmpv6_echo_ignore_all, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { }, }; @@ -1115,9 +1124,10 @@ struct ctl_table * __net_init ipv6_icmp_sysctl_init(struct net *net) sizeof(ipv6_icmp_table_template), GFP_KERNEL); - if (table) + if (table) { table[0].data = &net->ipv6.sysctl.icmpv6_time; - + table[1].data = &net->ipv6.sysctl.icmpv6_echo_ignore_all; + } return table; } #endif diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 595ad408dba0..3d7c7460a0c5 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -191,7 +191,7 @@ struct sock *inet6_lookup_listener(struct net *net, saddr, sport, daddr, hnum, dif, sdif); if (result) - return result; + goto done; /* Lookup lhash2 with in6addr_any */ @@ -200,9 +200,10 @@ struct sock *inet6_lookup_listener(struct net *net, if (ilb2->count > ilb->count) goto port_lookup; - return inet6_lhash2_lookup(net, ilb2, skb, doff, - saddr, sport, daddr, hnum, - dif, sdif); + result = inet6_lhash2_lookup(net, ilb2, skb, doff, + saddr, sport, daddr, hnum, + dif, sdif); + goto done; port_lookup: sk_for_each(sk, &ilb->head) { @@ -214,12 +215,15 @@ port_lookup: result = reuseport_select_sock(sk, phash, skb, doff); if (result) - return result; + goto done; } result = sk; hiscore = score; } } +done: + if (unlikely(IS_ERR(result))) + return NULL; return result; } EXPORT_SYMBOL_GPL(inet6_lookup_listener); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index f6b96956a8ed..83f4c77c79d8 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -235,6 +235,8 @@ struct sock *__udp6_lib_lookup(struct net *net, exact_dif, hslot2, skb); } + if (unlikely(IS_ERR(result))) + return NULL; return result; } begin: @@ -249,6 +251,8 @@ begin: saddr, sport); result = reuseport_select_sock(sk, hash, skb, sizeof(struct udphdr)); + if (unlikely(IS_ERR(result))) + return NULL; if (result) return result; } diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c index ac6a00bcec71..82cdf9020b53 100644 --- a/net/l2tp/l2tp_core.c +++ b/net/l2tp/l2tp_core.c @@ -203,44 +203,44 @@ struct l2tp_tunnel *l2tp_tunnel_get_nth(const struct net *net, int nth) } EXPORT_SYMBOL_GPL(l2tp_tunnel_get_nth); -/* Lookup a session. A new reference is held on the returned session. */ -struct l2tp_session *l2tp_session_get(const struct net *net, - struct l2tp_tunnel *tunnel, - u32 session_id) +struct l2tp_session *l2tp_tunnel_get_session(struct l2tp_tunnel *tunnel, + u32 session_id) { struct hlist_head *session_list; struct l2tp_session *session; - if (!tunnel) { - struct l2tp_net *pn = l2tp_pernet(net); - - session_list = l2tp_session_id_hash_2(pn, session_id); + session_list = l2tp_session_id_hash(tunnel, session_id); - rcu_read_lock_bh(); - hlist_for_each_entry_rcu(session, session_list, global_hlist) { - if (session->session_id == session_id) { - l2tp_session_inc_refcount(session); - rcu_read_unlock_bh(); + read_lock_bh(&tunnel->hlist_lock); + hlist_for_each_entry(session, session_list, hlist) + if (session->session_id == session_id) { + l2tp_session_inc_refcount(session); + read_unlock_bh(&tunnel->hlist_lock); - return session; - } + return session; } - rcu_read_unlock_bh(); + read_unlock_bh(&tunnel->hlist_lock); - return NULL; - } + return NULL; +} +EXPORT_SYMBOL_GPL(l2tp_tunnel_get_session); - session_list = l2tp_session_id_hash(tunnel, session_id); - read_lock_bh(&tunnel->hlist_lock); - hlist_for_each_entry(session, session_list, hlist) { +struct l2tp_session *l2tp_session_get(const struct net *net, u32 session_id) +{ + struct hlist_head *session_list; + struct l2tp_session *session; + + session_list = l2tp_session_id_hash_2(l2tp_pernet(net), session_id); + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu(session, session_list, global_hlist) if (session->session_id == session_id) { l2tp_session_inc_refcount(session); - read_unlock_bh(&tunnel->hlist_lock); + rcu_read_unlock_bh(); return session; } - } - read_unlock_bh(&tunnel->hlist_lock); + rcu_read_unlock_bh(); return NULL; } @@ -872,7 +872,7 @@ static int l2tp_udp_recv_core(struct l2tp_tunnel *tunnel, struct sk_buff *skb) } /* Find the session context */ - session = l2tp_session_get(tunnel->l2tp_net, tunnel, session_id); + session = l2tp_tunnel_get_session(tunnel, session_id); if (!session || !session->recv_skb) { if (session) l2tp_session_dec_refcount(session); @@ -1098,7 +1098,7 @@ int l2tp_xmit_skb(struct l2tp_session *session, struct sk_buff *skb, int hdr_len /* Get routing info from the tunnel socket */ skb_dst_drop(skb); - skb_dst_set(skb, dst_clone(__sk_dst_check(sk, 0))); + skb_dst_set(skb, sk_dst_check(sk, 0)); inet = inet_sk(sk); fl = &inet->cork.fl; diff --git a/net/l2tp/l2tp_core.h b/net/l2tp/l2tp_core.h index 5804065dfbfb..8480a0af973e 100644 --- a/net/l2tp/l2tp_core.h +++ b/net/l2tp/l2tp_core.h @@ -15,6 +15,10 @@ #include <net/dst.h> #include <net/sock.h> +#ifdef CONFIG_XFRM +#include <net/xfrm.h> +#endif + /* Just some random numbers */ #define L2TP_TUNNEL_MAGIC 0x42114DDA #define L2TP_SESSION_MAGIC 0x0C04EB7D @@ -192,12 +196,12 @@ static inline void *l2tp_session_priv(struct l2tp_session *session) struct l2tp_tunnel *l2tp_tunnel_get(const struct net *net, u32 tunnel_id); struct l2tp_tunnel *l2tp_tunnel_get_nth(const struct net *net, int nth); +struct l2tp_session *l2tp_tunnel_get_session(struct l2tp_tunnel *tunnel, + u32 session_id); void l2tp_tunnel_free(struct l2tp_tunnel *tunnel); -struct l2tp_session *l2tp_session_get(const struct net *net, - struct l2tp_tunnel *tunnel, - u32 session_id); +struct l2tp_session *l2tp_session_get(const struct net *net, u32 session_id); struct l2tp_session *l2tp_session_get_nth(struct l2tp_tunnel *tunnel, int nth); struct l2tp_session *l2tp_session_get_by_ifname(const struct net *net, const char *ifname); @@ -284,6 +288,21 @@ static inline u32 l2tp_tunnel_dst_mtu(const struct l2tp_tunnel *tunnel) return mtu; } +#ifdef CONFIG_XFRM +static inline bool l2tp_tunnel_uses_xfrm(const struct l2tp_tunnel *tunnel) +{ + struct sock *sk = tunnel->sock; + + return sk && (rcu_access_pointer(sk->sk_policy[0]) || + rcu_access_pointer(sk->sk_policy[1])); +} +#else +static inline bool l2tp_tunnel_uses_xfrm(const struct l2tp_tunnel *tunnel) +{ + return false; +} +#endif + #define l2tp_printk(ptr, type, func, fmt, ...) \ do { \ if (((ptr)->debug) & (type)) \ diff --git a/net/l2tp/l2tp_ip.c b/net/l2tp/l2tp_ip.c index 0bc39cc20a3f..35f6f86d4dcc 100644 --- a/net/l2tp/l2tp_ip.c +++ b/net/l2tp/l2tp_ip.c @@ -144,7 +144,7 @@ static int l2tp_ip_recv(struct sk_buff *skb) } /* Ok, this is a data packet. Lookup the session. */ - session = l2tp_session_get(net, NULL, session_id); + session = l2tp_session_get(net, session_id); if (!session) goto discard; diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c index 42f828cf62fb..237f1a4a0b0c 100644 --- a/net/l2tp/l2tp_ip6.c +++ b/net/l2tp/l2tp_ip6.c @@ -157,7 +157,7 @@ static int l2tp_ip6_recv(struct sk_buff *skb) } /* Ok, this is a data packet. Lookup the session. */ - session = l2tp_session_get(net, NULL, session_id); + session = l2tp_session_get(net, session_id); if (!session) goto discard; diff --git a/net/l2tp/l2tp_netlink.c b/net/l2tp/l2tp_netlink.c index 2e1e92651545..edbd5d1fbcde 100644 --- a/net/l2tp/l2tp_netlink.c +++ b/net/l2tp/l2tp_netlink.c @@ -66,7 +66,7 @@ static struct l2tp_session *l2tp_nl_session_get(struct genl_info *info) session_id = nla_get_u32(info->attrs[L2TP_ATTR_SESSION_ID]); tunnel = l2tp_tunnel_get(net, tunnel_id); if (tunnel) { - session = l2tp_session_get(net, tunnel, session_id); + session = l2tp_tunnel_get_session(tunnel, session_id); l2tp_tunnel_dec_refcount(tunnel); } } @@ -627,7 +627,7 @@ static int l2tp_nl_cmd_session_create(struct sk_buff *skb, struct genl_info *inf &cfg); if (ret >= 0) { - session = l2tp_session_get(net, tunnel, session_id); + session = l2tp_tunnel_get_session(tunnel, session_id); if (session) { ret = l2tp_session_notify(&l2tp_nl_family, info, session, L2TP_CMD_SESSION_CREATE); @@ -710,9 +710,6 @@ static int l2tp_nl_session_send(struct sk_buff *skb, u32 portid, u32 seq, int fl void *hdr; struct nlattr *nest; struct l2tp_tunnel *tunnel = session->tunnel; - struct sock *sk = NULL; - - sk = tunnel->sock; hdr = genlmsg_put(skb, portid, seq, &l2tp_nl_family, flags, cmd); if (!hdr) @@ -738,10 +735,8 @@ static int l2tp_nl_session_send(struct sk_buff *skb, u32 portid, u32 seq, int fl nla_put_u8(skb, L2TP_ATTR_RECV_SEQ, session->recv_seq) || nla_put_u8(skb, L2TP_ATTR_SEND_SEQ, session->send_seq) || nla_put_u8(skb, L2TP_ATTR_LNS_MODE, session->lns_mode) || -#ifdef CONFIG_XFRM - (((sk) && (sk->sk_policy[0] || sk->sk_policy[1])) && + (l2tp_tunnel_uses_xfrm(tunnel) && nla_put_u8(skb, L2TP_ATTR_USING_IPSEC, 1)) || -#endif (session->reorder_timeout && nla_put_msecs(skb, L2TP_ATTR_RECV_TIMEOUT, session->reorder_timeout, L2TP_ATTR_PAD))) diff --git a/net/l2tp/l2tp_ppp.c b/net/l2tp/l2tp_ppp.c index 6e2c8e7595e0..62f2d3f1e431 100644 --- a/net/l2tp/l2tp_ppp.c +++ b/net/l2tp/l2tp_ppp.c @@ -95,7 +95,6 @@ #include <net/netns/generic.h> #include <net/ip.h> #include <net/udp.h> -#include <net/xfrm.h> #include <net/inet_common.h> #include <asm/byteorder.h> @@ -758,7 +757,7 @@ static int pppol2tp_connect(struct socket *sock, struct sockaddr *uservaddr, if (tunnel->peer_tunnel_id == 0) tunnel->peer_tunnel_id = info.peer_tunnel_id; - session = l2tp_session_get(sock_net(sk), tunnel, info.session_id); + session = l2tp_tunnel_get_session(tunnel, info.session_id); if (session) { drop_refcnt = true; @@ -1027,8 +1026,10 @@ end: ****************************************************************************/ static void pppol2tp_copy_stats(struct pppol2tp_ioc_stats *dest, - struct l2tp_stats *stats) + const struct l2tp_stats *stats) { + memset(dest, 0, sizeof(*dest)); + dest->tx_packets = atomic_long_read(&stats->tx_packets); dest->tx_bytes = atomic_long_read(&stats->tx_bytes); dest->tx_errors = atomic_long_read(&stats->tx_errors); @@ -1039,188 +1040,107 @@ static void pppol2tp_copy_stats(struct pppol2tp_ioc_stats *dest, dest->rx_errors = atomic_long_read(&stats->rx_errors); } -/* Session ioctl helper. - */ -static int pppol2tp_session_ioctl(struct l2tp_session *session, - unsigned int cmd, unsigned long arg) +static int pppol2tp_tunnel_copy_stats(struct pppol2tp_ioc_stats *stats, + struct l2tp_tunnel *tunnel) { - int err = 0; - struct sock *sk; - int val = (int) arg; - struct l2tp_tunnel *tunnel = session->tunnel; - struct pppol2tp_ioc_stats stats; + struct l2tp_session *session; - l2tp_dbg(session, L2TP_MSG_CONTROL, - "%s: pppol2tp_session_ioctl(cmd=%#x, arg=%#lx)\n", - session->name, cmd, arg); + if (!stats->session_id) { + pppol2tp_copy_stats(stats, &tunnel->stats); + return 0; + } - sk = pppol2tp_session_get_sock(session); - if (!sk) + /* If session_id is set, search the corresponding session in the + * context of this tunnel and record the session's statistics. + */ + session = l2tp_tunnel_get_session(tunnel, stats->session_id); + if (!session) return -EBADR; - switch (cmd) { - case PPPIOCGMRU: - case PPPIOCGFLAGS: - err = -EFAULT; - if (put_user(0, (int __user *)arg)) - break; - err = 0; - break; - - case PPPIOCSMRU: - case PPPIOCSFLAGS: - err = -EFAULT; - if (get_user(val, (int __user *)arg)) - break; - err = 0; - break; - - case PPPIOCGL2TPSTATS: - err = -ENXIO; - if (!(sk->sk_state & PPPOX_CONNECTED)) - break; - - memset(&stats, 0, sizeof(stats)); - stats.tunnel_id = tunnel->tunnel_id; - stats.session_id = session->session_id; - pppol2tp_copy_stats(&stats, &session->stats); - if (copy_to_user((void __user *) arg, &stats, - sizeof(stats))) - break; - l2tp_info(session, L2TP_MSG_CONTROL, "%s: get L2TP stats\n", - session->name); - err = 0; - break; - - default: - err = -ENOSYS; - break; + if (session->pwtype != L2TP_PWTYPE_PPP) { + l2tp_session_dec_refcount(session); + return -EBADR; } - sock_put(sk); + pppol2tp_copy_stats(stats, &session->stats); + l2tp_session_dec_refcount(session); - return err; + return 0; } -/* Tunnel ioctl helper. - * - * Note the special handling for PPPIOCGL2TPSTATS below. If the ioctl data - * specifies a session_id, the session ioctl handler is called. This allows an - * application to retrieve session stats via a tunnel socket. - */ -static int pppol2tp_tunnel_ioctl(struct l2tp_tunnel *tunnel, - unsigned int cmd, unsigned long arg) +static int pppol2tp_ioctl(struct socket *sock, unsigned int cmd, + unsigned long arg) { - int err = 0; - struct sock *sk; struct pppol2tp_ioc_stats stats; - - l2tp_dbg(tunnel, L2TP_MSG_CONTROL, - "%s: pppol2tp_tunnel_ioctl(cmd=%#x, arg=%#lx)\n", - tunnel->name, cmd, arg); - - sk = tunnel->sock; - sock_hold(sk); + struct l2tp_session *session; + int val; switch (cmd) { - case PPPIOCGL2TPSTATS: - err = -ENXIO; - if (!(sk->sk_state & PPPOX_CONNECTED)) - break; - - if (copy_from_user(&stats, (void __user *) arg, - sizeof(stats))) { - err = -EFAULT; - break; - } - if (stats.session_id != 0) { - /* resend to session ioctl handler */ - struct l2tp_session *session = - l2tp_session_get(sock_net(sk), tunnel, - stats.session_id); - - if (!session) { - err = -EBADR; - break; - } - if (session->pwtype != L2TP_PWTYPE_PPP) { - l2tp_session_dec_refcount(session); - err = -EBADR; - break; - } + case PPPIOCGMRU: + case PPPIOCGFLAGS: + session = sock->sk->sk_user_data; + if (!session) + return -ENOTCONN; - err = pppol2tp_session_ioctl(session, cmd, arg); - l2tp_session_dec_refcount(session); - break; - } -#ifdef CONFIG_XFRM - stats.using_ipsec = (sk->sk_policy[0] || sk->sk_policy[1]) ? 1 : 0; -#endif - pppol2tp_copy_stats(&stats, &tunnel->stats); - if (copy_to_user((void __user *) arg, &stats, sizeof(stats))) { - err = -EFAULT; - break; - } - l2tp_info(tunnel, L2TP_MSG_CONTROL, "%s: get L2TP stats\n", - tunnel->name); - err = 0; - break; + /* Not defined for tunnels */ + if (!session->session_id && !session->peer_session_id) + return -ENOSYS; - default: - err = -ENOSYS; + if (put_user(0, (int __user *)arg)) + return -EFAULT; break; - } - - sock_put(sk); - - return err; -} -/* Main ioctl() handler. - * Dispatch to tunnel or session helpers depending on the socket. - */ -static int pppol2tp_ioctl(struct socket *sock, unsigned int cmd, - unsigned long arg) -{ - struct sock *sk = sock->sk; - struct l2tp_session *session; - struct l2tp_tunnel *tunnel; - int err; + case PPPIOCSMRU: + case PPPIOCSFLAGS: + session = sock->sk->sk_user_data; + if (!session) + return -ENOTCONN; - if (!sk) - return 0; + /* Not defined for tunnels */ + if (!session->session_id && !session->peer_session_id) + return -ENOSYS; - err = -EBADF; - if (sock_flag(sk, SOCK_DEAD) != 0) - goto end; + if (get_user(val, (int __user *)arg)) + return -EFAULT; + break; - err = -ENOTCONN; - if ((sk->sk_user_data == NULL) || - (!(sk->sk_state & (PPPOX_CONNECTED | PPPOX_BOUND)))) - goto end; + case PPPIOCGL2TPSTATS: + session = sock->sk->sk_user_data; + if (!session) + return -ENOTCONN; + + /* Session 0 represents the parent tunnel */ + if (!session->session_id && !session->peer_session_id) { + u32 session_id; + int err; + + if (copy_from_user(&stats, (void __user *)arg, + sizeof(stats))) + return -EFAULT; + + session_id = stats.session_id; + err = pppol2tp_tunnel_copy_stats(&stats, + session->tunnel); + if (err < 0) + return err; + + stats.session_id = session_id; + } else { + pppol2tp_copy_stats(&stats, &session->stats); + stats.session_id = session->session_id; + } + stats.tunnel_id = session->tunnel->tunnel_id; + stats.using_ipsec = l2tp_tunnel_uses_xfrm(session->tunnel); - /* Get session context from the socket */ - err = -EBADF; - session = pppol2tp_sock_to_session(sk); - if (session == NULL) - goto end; + if (copy_to_user((void __user *)arg, &stats, sizeof(stats))) + return -EFAULT; + break; - /* Special case: if session's session_id is zero, treat ioctl as a - * tunnel ioctl - */ - if ((session->session_id == 0) && - (session->peer_session_id == 0)) { - tunnel = session->tunnel; - err = pppol2tp_tunnel_ioctl(tunnel, cmd, arg); - goto end_put_sess; + default: + return -ENOIOCTLCMD; } - err = pppol2tp_session_ioctl(session, cmd, arg); - -end_put_sess: - sock_put(sk); -end: - return err; + return 0; } /***************************************************************************** diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 75c92a87e7b2..5610061e7f2e 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -4137,52 +4137,36 @@ static const struct vm_operations_struct packet_mmap_ops = { .close = packet_mm_close, }; -static void free_pg_vec(struct pgv *pg_vec, unsigned int order, - unsigned int len) +static void free_pg_vec(struct pgv *pg_vec, unsigned int len) { int i; for (i = 0; i < len; i++) { if (likely(pg_vec[i].buffer)) { - if (is_vmalloc_addr(pg_vec[i].buffer)) - vfree(pg_vec[i].buffer); - else - free_pages((unsigned long)pg_vec[i].buffer, - order); + kvfree(pg_vec[i].buffer); pg_vec[i].buffer = NULL; } } kfree(pg_vec); } -static char *alloc_one_pg_vec_page(unsigned long order) +static char *alloc_one_pg_vec_page(unsigned long size) { char *buffer; - gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | - __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY; - buffer = (char *) __get_free_pages(gfp_flags, order); + buffer = kvzalloc(size, GFP_KERNEL); if (buffer) return buffer; - /* __get_free_pages failed, fall back to vmalloc */ - buffer = vzalloc(array_size((1 << order), PAGE_SIZE)); - if (buffer) - return buffer; + buffer = kvzalloc(size, GFP_KERNEL | __GFP_RETRY_MAYFAIL); - /* vmalloc failed, lets dig into swap here */ - gfp_flags &= ~__GFP_NORETRY; - buffer = (char *) __get_free_pages(gfp_flags, order); - if (buffer) - return buffer; - - /* complete and utter failure */ - return NULL; + return buffer; } -static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order) +static struct pgv *alloc_pg_vec(struct tpacket_req *req) { unsigned int block_nr = req->tp_block_nr; + unsigned long size = req->tp_block_size; struct pgv *pg_vec; int i; @@ -4191,7 +4175,7 @@ static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order) goto out; for (i = 0; i < block_nr; i++) { - pg_vec[i].buffer = alloc_one_pg_vec_page(order); + pg_vec[i].buffer = alloc_one_pg_vec_page(size); if (unlikely(!pg_vec[i].buffer)) goto out_free_pgvec; } @@ -4200,7 +4184,7 @@ out: return pg_vec; out_free_pgvec: - free_pg_vec(pg_vec, order, block_nr); + free_pg_vec(pg_vec, block_nr); pg_vec = NULL; goto out; } @@ -4210,9 +4194,9 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, { struct pgv *pg_vec = NULL; struct packet_sock *po = pkt_sk(sk); - int was_running, order = 0; struct packet_ring_buffer *rb; struct sk_buff_head *rb_queue; + int was_running; __be16 num; int err = -EINVAL; /* Added to avoid minimal code churn */ @@ -4274,8 +4258,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, goto out; err = -ENOMEM; - order = get_order(req->tp_block_size); - pg_vec = alloc_pg_vec(req, order); + pg_vec = alloc_pg_vec(req); if (unlikely(!pg_vec)) goto out; switch (po->tp_version) { @@ -4329,7 +4312,6 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, rb->frame_size = req->tp_frame_size; spin_unlock_bh(&rb_queue->lock); - swap(rb->pg_vec_order, order); swap(rb->pg_vec_len, req->tp_block_nr); rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE; @@ -4355,7 +4337,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, } if (pg_vec) - free_pg_vec(pg_vec, order, req->tp_block_nr); + free_pg_vec(pg_vec, req->tp_block_nr); out: return err; } diff --git a/net/packet/internal.h b/net/packet/internal.h index 3bb7c5fb3bff..8f50036f62f0 100644 --- a/net/packet/internal.h +++ b/net/packet/internal.h @@ -64,7 +64,6 @@ struct packet_ring_buffer { unsigned int frame_size; unsigned int frame_max; - unsigned int pg_vec_order; unsigned int pg_vec_pages; unsigned int pg_vec_len; diff --git a/net/rxrpc/sysctl.c b/net/rxrpc/sysctl.c index 4a7af7aff37d..d75bd15151e6 100644 --- a/net/rxrpc/sysctl.c +++ b/net/rxrpc/sysctl.c @@ -15,7 +15,6 @@ #include "ar-internal.h" static struct ctl_table_header *rxrpc_sysctl_reg_table; -static const unsigned int zero = 0; static const unsigned int one = 1; static const unsigned int four = 4; static const unsigned int thirtytwo = 32; diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index 6203eb075c9a..9b30e62805c7 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -34,8 +34,8 @@ struct tcf_bpf_cfg { static unsigned int bpf_net_id; static struct tc_action_ops act_bpf_ops; -static int tcf_bpf(struct sk_buff *skb, const struct tc_action *act, - struct tcf_result *res) +static int tcf_bpf_act(struct sk_buff *skb, const struct tc_action *act, + struct tcf_result *res) { bool at_ingress = skb_at_tc_ingress(skb); struct tcf_bpf *prog = to_bpf(act); @@ -143,11 +143,12 @@ static int tcf_bpf_dump(struct sk_buff *skb, struct tc_action *act, .index = prog->tcf_index, .refcnt = refcount_read(&prog->tcf_refcnt) - ref, .bindcnt = atomic_read(&prog->tcf_bindcnt) - bind, - .action = prog->tcf_action, }; struct tcf_t tm; int ret; + spin_lock(&prog->tcf_lock); + opt.action = prog->tcf_action; if (nla_put(skb, TCA_ACT_BPF_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -163,9 +164,11 @@ static int tcf_bpf_dump(struct sk_buff *skb, struct tc_action *act, TCA_ACT_BPF_PAD)) goto nla_put_failure; + spin_unlock(&prog->tcf_lock); return skb->len; nla_put_failure: + spin_unlock(&prog->tcf_lock); nlmsg_trim(skb, tp); return -1; } @@ -264,7 +267,7 @@ static void tcf_bpf_prog_fill_cfg(const struct tcf_bpf *prog, { cfg->is_ebpf = tcf_bpf_is_ebpf(prog); /* updates to prog->filter are prevented, since it's called either - * with rtnl lock or during final cleanup in rcu callback + * with tcf lock or during final cleanup in rcu callback */ cfg->filter = rcu_dereference_protected(prog->filter, 1); @@ -336,8 +339,8 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla, goto out; prog = to_bpf(*act); - ASSERT_RTNL(); + spin_lock(&prog->tcf_lock); if (res != ACT_P_CREATED) tcf_bpf_prog_fill_cfg(prog, &old); @@ -349,6 +352,7 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla, prog->tcf_action = parm->action; rcu_assign_pointer(prog->filter, cfg.filter); + spin_unlock(&prog->tcf_lock); if (res == ACT_P_CREATED) { tcf_idr_insert(tn, *act); @@ -402,7 +406,7 @@ static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .type = TCA_ACT_BPF, .owner = THIS_MODULE, - .act = tcf_bpf, + .act = tcf_bpf_act, .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c index 2f9bc833d046..54c0bf54f2ac 100644 --- a/net/sched/act_connmark.c +++ b/net/sched/act_connmark.c @@ -31,8 +31,8 @@ static unsigned int connmark_net_id; static struct tc_action_ops act_connmark_ops; -static int tcf_connmark(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_connmark_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { const struct nf_conntrack_tuple_hash *thash; struct nf_conntrack_tuple tuple; @@ -209,7 +209,7 @@ static struct tc_action_ops act_connmark_ops = { .kind = "connmark", .type = TCA_ACT_CONNMARK, .owner = THIS_MODULE, - .act = tcf_connmark, + .act = tcf_connmark_act, .dump = tcf_connmark_dump, .init = tcf_connmark_init, .walk = tcf_connmark_walker, diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 648a3a35b720..5596fae4e478 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -50,7 +50,7 @@ static int tcf_csum_init(struct net *net, struct nlattr *nla, struct netlink_ext_ack *extack) { struct tc_action_net *tn = net_generic(net, csum_net_id); - struct tcf_csum_params *params_old, *params_new; + struct tcf_csum_params *params_new; struct nlattr *tb[TCA_CSUM_MAX + 1]; struct tc_csum *parm; struct tcf_csum *p; @@ -88,20 +88,22 @@ static int tcf_csum_init(struct net *net, struct nlattr *nla, } p = to_tcf_csum(*a); - ASSERT_RTNL(); params_new = kzalloc(sizeof(*params_new), GFP_KERNEL); if (unlikely(!params_new)) { tcf_idr_release(*a, bind); return -ENOMEM; } - params_old = rtnl_dereference(p->params); + params_new->update_flags = parm->update_flags; + spin_lock(&p->tcf_lock); p->tcf_action = parm->action; - params_new->update_flags = parm->update_flags; - rcu_assign_pointer(p->params, params_new); - if (params_old) - kfree_rcu(params_old, rcu); + rcu_swap_protected(p->params, params_new, + lockdep_is_held(&p->tcf_lock)); + spin_unlock(&p->tcf_lock); + + if (params_new) + kfree_rcu(params_new, rcu); if (ret == ACT_P_CREATED) tcf_idr_insert(tn, *a); @@ -553,8 +555,8 @@ fail: return 0; } -static int tcf_csum(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_csum_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_csum *p = to_tcf_csum(a); struct tcf_csum_params *params; @@ -599,11 +601,13 @@ static int tcf_csum_dump(struct sk_buff *skb, struct tc_action *a, int bind, .index = p->tcf_index, .refcnt = refcount_read(&p->tcf_refcnt) - ref, .bindcnt = atomic_read(&p->tcf_bindcnt) - bind, - .action = p->tcf_action, }; struct tcf_t t; - params = rtnl_dereference(p->params); + spin_lock(&p->tcf_lock); + params = rcu_dereference_protected(p->params, + lockdep_is_held(&p->tcf_lock)); + opt.action = p->tcf_action; opt.update_flags = params->update_flags; if (nla_put(skb, TCA_CSUM_PARMS, sizeof(opt), &opt)) @@ -612,10 +616,12 @@ static int tcf_csum_dump(struct sk_buff *skb, struct tc_action *a, int bind, tcf_tm_dump(&t, &p->tcf_tm); if (nla_put_64bit(skb, TCA_CSUM_TM, sizeof(t), &t, TCA_CSUM_PAD)) goto nla_put_failure; + spin_unlock(&p->tcf_lock); return skb->len; nla_put_failure: + spin_unlock(&p->tcf_lock); nlmsg_trim(skb, b); return -1; } @@ -664,7 +670,7 @@ static struct tc_action_ops act_csum_ops = { .kind = "csum", .type = TCA_ACT_CSUM, .owner = THIS_MODULE, - .act = tcf_csum, + .act = tcf_csum_act, .dump = tcf_csum_dump, .init = tcf_csum_init, .cleanup = tcf_csum_cleanup, diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c index 661b72b9147d..52a3e474d822 100644 --- a/net/sched/act_gact.c +++ b/net/sched/act_gact.c @@ -113,7 +113,7 @@ static int tcf_gact_init(struct net *net, struct nlattr *nla, gact = to_gact(*a); - ASSERT_RTNL(); + spin_lock(&gact->tcf_lock); gact->tcf_action = parm->action; #ifdef CONFIG_GACT_PROB if (p_parm) { @@ -126,13 +126,15 @@ static int tcf_gact_init(struct net *net, struct nlattr *nla, gact->tcfg_ptype = p_parm->ptype; } #endif + spin_unlock(&gact->tcf_lock); + if (ret == ACT_P_CREATED) tcf_idr_insert(tn, *a); return ret; } -static int tcf_gact(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_gact_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_gact *gact = to_gact(a); int action = READ_ONCE(gact->tcf_action); @@ -178,10 +180,11 @@ static int tcf_gact_dump(struct sk_buff *skb, struct tc_action *a, .index = gact->tcf_index, .refcnt = refcount_read(&gact->tcf_refcnt) - ref, .bindcnt = atomic_read(&gact->tcf_bindcnt) - bind, - .action = gact->tcf_action, }; struct tcf_t t; + spin_lock(&gact->tcf_lock); + opt.action = gact->tcf_action; if (nla_put(skb, TCA_GACT_PARMS, sizeof(opt), &opt)) goto nla_put_failure; #ifdef CONFIG_GACT_PROB @@ -199,9 +202,12 @@ static int tcf_gact_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &gact->tcf_tm); if (nla_put_64bit(skb, TCA_GACT_TM, sizeof(t), &t, TCA_GACT_PAD)) goto nla_put_failure; + spin_unlock(&gact->tcf_lock); + return skb->len; nla_put_failure: + spin_unlock(&gact->tcf_lock); nlmsg_trim(skb, b); return -1; } @@ -248,7 +254,7 @@ static struct tc_action_ops act_gact_ops = { .kind = "gact", .type = TCA_ACT_GACT, .owner = THIS_MODULE, - .act = tcf_gact, + .act = tcf_gact_act, .stats_update = tcf_gact_stats_update, .dump = tcf_gact_dump, .init = tcf_gact_init, diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index df4060e32d43..5d200495e467 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -268,7 +268,8 @@ static const char *ife_meta_id2name(u32 metaid) * under ife->tcf_lock for existing action */ static int load_metaops_and_vet(struct tcf_ife_info *ife, u32 metaid, - void *val, int len, bool exists) + void *val, int len, bool exists, + bool rtnl_held) { struct tcf_meta_ops *ops = find_ife_oplist(metaid); int ret = 0; @@ -278,9 +279,11 @@ static int load_metaops_and_vet(struct tcf_ife_info *ife, u32 metaid, #ifdef CONFIG_MODULES if (exists) spin_unlock_bh(&ife->tcf_lock); - rtnl_unlock(); + if (rtnl_held) + rtnl_unlock(); request_module("ife-meta-%s", ife_meta_id2name(metaid)); - rtnl_lock(); + if (rtnl_held) + rtnl_lock(); if (exists) spin_lock_bh(&ife->tcf_lock); ops = find_ife_oplist(metaid); @@ -421,7 +424,7 @@ static void tcf_ife_cleanup(struct tc_action *a) /* under ife->tcf_lock for existing action */ static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, - bool exists) + bool exists, bool rtnl_held) { int len = 0; int rc = 0; @@ -433,7 +436,8 @@ static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, val = nla_data(tb[i]); len = nla_len(tb[i]); - rc = load_metaops_and_vet(ife, i, val, len, exists); + rc = load_metaops_and_vet(ife, i, val, len, exists, + rtnl_held); if (rc != 0) return rc; @@ -454,7 +458,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, struct tc_action_net *tn = net_generic(net, ife_net_id); struct nlattr *tb[TCA_IFE_MAX + 1]; struct nlattr *tb2[IFE_META_MAX + 1]; - struct tcf_ife_params *p, *p_old; + struct tcf_ife_params *p; struct tcf_ife_info *ife; u16 ife_type = ETH_P_IFE; struct tc_ife *parm; @@ -558,7 +562,7 @@ metadata_parse_err: return err; } - err = populate_metalist(ife, tb2, exists); + err = populate_metalist(ife, tb2, exists, rtnl_held); if (err) goto metadata_parse_err; @@ -581,13 +585,13 @@ metadata_parse_err: } ife->tcf_action = parm->action; + /* protected by tcf_lock when modifying existing action */ + rcu_swap_protected(ife->params, p, 1); + if (exists) spin_unlock_bh(&ife->tcf_lock); - - p_old = rtnl_dereference(ife->params); - rcu_assign_pointer(ife->params, p); - if (p_old) - kfree_rcu(p_old, rcu); + if (p) + kfree_rcu(p, rcu); if (ret == ACT_P_CREATED) tcf_idr_insert(tn, *a); @@ -600,16 +604,20 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, { unsigned char *b = skb_tail_pointer(skb); struct tcf_ife_info *ife = to_ife(a); - struct tcf_ife_params *p = rtnl_dereference(ife->params); + struct tcf_ife_params *p; struct tc_ife opt = { .index = ife->tcf_index, .refcnt = refcount_read(&ife->tcf_refcnt) - ref, .bindcnt = atomic_read(&ife->tcf_bindcnt) - bind, - .action = ife->tcf_action, - .flags = p->flags, }; struct tcf_t t; + spin_lock_bh(&ife->tcf_lock); + opt.action = ife->tcf_action; + p = rcu_dereference_protected(ife->params, + lockdep_is_held(&ife->tcf_lock)); + opt.flags = p->flags; + if (nla_put(skb, TCA_IFE_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -635,9 +643,11 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, pr_info("Failed to dump metalist\n"); } + spin_unlock_bh(&ife->tcf_lock); return skb->len; nla_put_failure: + spin_unlock_bh(&ife->tcf_lock); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_ipt.c b/net/sched/act_ipt.c index 0dc787a57798..51f235bbeb5b 100644 --- a/net/sched/act_ipt.c +++ b/net/sched/act_ipt.c @@ -222,8 +222,8 @@ static int tcf_xt_init(struct net *net, struct nlattr *nla, bind); } -static int tcf_ipt(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_ipt_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { int ret = 0, result = 0; struct tcf_ipt *ipt = to_ipt(a); @@ -288,6 +288,7 @@ static int tcf_ipt_dump(struct sk_buff *skb, struct tc_action *a, int bind, * for foolproof you need to not assume this */ + spin_lock_bh(&ipt->tcf_lock); t = kmemdup(ipt->tcfi_t, ipt->tcfi_t->u.user.target_size, GFP_ATOMIC); if (unlikely(!t)) goto nla_put_failure; @@ -307,10 +308,12 @@ static int tcf_ipt_dump(struct sk_buff *skb, struct tc_action *a, int bind, if (nla_put_64bit(skb, TCA_IPT_TM, sizeof(tm), &tm, TCA_IPT_PAD)) goto nla_put_failure; + spin_unlock_bh(&ipt->tcf_lock); kfree(t); return skb->len; nla_put_failure: + spin_unlock_bh(&ipt->tcf_lock); nlmsg_trim(skb, b); kfree(t); return -1; @@ -345,7 +348,7 @@ static struct tc_action_ops act_ipt_ops = { .kind = "ipt", .type = TCA_ACT_IPT, .owner = THIS_MODULE, - .act = tcf_ipt, + .act = tcf_ipt_act, .dump = tcf_ipt_dump, .cleanup = tcf_ipt_release, .init = tcf_ipt_init, @@ -403,7 +406,7 @@ static struct tc_action_ops act_xt_ops = { .kind = "xt", .type = TCA_ACT_XT, .owner = THIS_MODULE, - .act = tcf_ipt, + .act = tcf_ipt_act, .dump = tcf_ipt_dump, .cleanup = tcf_ipt_release, .init = tcf_xt_init, diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index b26d060da08e..8ec216001077 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -30,6 +30,7 @@ #include <net/tc_act/tc_mirred.h> static LIST_HEAD(mirred_list); +static DEFINE_SPINLOCK(mirred_list_lock); static bool tcf_mirred_is_act_redirect(int action) { @@ -62,13 +63,23 @@ static bool tcf_mirred_can_reinsert(int action) return false; } +static struct net_device *tcf_mirred_dev_dereference(struct tcf_mirred *m) +{ + return rcu_dereference_protected(m->tcfm_dev, + lockdep_is_held(&m->tcf_lock)); +} + static void tcf_mirred_release(struct tc_action *a) { struct tcf_mirred *m = to_mirred(a); struct net_device *dev; + spin_lock(&mirred_list_lock); list_del(&m->tcfm_list); - dev = rtnl_dereference(m->tcfm_dev); + spin_unlock(&mirred_list_lock); + + /* last reference to action, no need to lock */ + dev = rcu_dereference_protected(m->tcfm_dev, 1); if (dev) dev_put(dev); } @@ -128,22 +139,9 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla, NL_SET_ERR_MSG_MOD(extack, "Unknown mirred option"); return -EINVAL; } - if (parm->ifindex) { - dev = __dev_get_by_index(net, parm->ifindex); - if (dev == NULL) { - if (exists) - tcf_idr_release(*a, bind); - else - tcf_idr_cleanup(tn, parm->index); - return -ENODEV; - } - mac_header_xmit = dev_is_mac_header_xmit(dev); - } else { - dev = NULL; - } if (!exists) { - if (!dev) { + if (!parm->ifindex) { tcf_idr_cleanup(tn, parm->index); NL_SET_ERR_MSG_MOD(extack, "Specified device does not exist"); return -EINVAL; @@ -161,27 +159,39 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla, } m = to_mirred(*a); - ASSERT_RTNL(); + spin_lock(&m->tcf_lock); m->tcf_action = parm->action; m->tcfm_eaction = parm->eaction; - if (dev != NULL) { - if (ret != ACT_P_CREATED) - dev_put(rcu_dereference_protected(m->tcfm_dev, 1)); - dev_hold(dev); - rcu_assign_pointer(m->tcfm_dev, dev); + + if (parm->ifindex) { + dev = dev_get_by_index(net, parm->ifindex); + if (!dev) { + spin_unlock(&m->tcf_lock); + tcf_idr_release(*a, bind); + return -ENODEV; + } + mac_header_xmit = dev_is_mac_header_xmit(dev); + rcu_swap_protected(m->tcfm_dev, dev, + lockdep_is_held(&m->tcf_lock)); + if (dev) + dev_put(dev); m->tcfm_mac_header_xmit = mac_header_xmit; } + spin_unlock(&m->tcf_lock); if (ret == ACT_P_CREATED) { + spin_lock(&mirred_list_lock); list_add(&m->tcfm_list, &mirred_list); + spin_unlock(&mirred_list_lock); + tcf_idr_insert(tn, *a); } return ret; } -static int tcf_mirred(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_mirred *m = to_mirred(a); struct sk_buff *skb2 = skb; @@ -287,26 +297,33 @@ static int tcf_mirred_dump(struct sk_buff *skb, struct tc_action *a, int bind, { unsigned char *b = skb_tail_pointer(skb); struct tcf_mirred *m = to_mirred(a); - struct net_device *dev = rtnl_dereference(m->tcfm_dev); struct tc_mirred opt = { .index = m->tcf_index, - .action = m->tcf_action, .refcnt = refcount_read(&m->tcf_refcnt) - ref, .bindcnt = atomic_read(&m->tcf_bindcnt) - bind, - .eaction = m->tcfm_eaction, - .ifindex = dev ? dev->ifindex : 0, }; + struct net_device *dev; struct tcf_t t; + spin_lock(&m->tcf_lock); + opt.action = m->tcf_action; + opt.eaction = m->tcfm_eaction; + dev = tcf_mirred_dev_dereference(m); + if (dev) + opt.ifindex = dev->ifindex; + if (nla_put(skb, TCA_MIRRED_PARMS, sizeof(opt), &opt)) goto nla_put_failure; tcf_tm_dump(&t, &m->tcf_tm); if (nla_put_64bit(skb, TCA_MIRRED_TM, sizeof(t), &t, TCA_MIRRED_PAD)) goto nla_put_failure; + spin_unlock(&m->tcf_lock); + return skb->len; nla_put_failure: + spin_unlock(&m->tcf_lock); nlmsg_trim(skb, b); return -1; } @@ -337,15 +354,19 @@ static int mirred_device_event(struct notifier_block *unused, ASSERT_RTNL(); if (event == NETDEV_UNREGISTER) { + spin_lock(&mirred_list_lock); list_for_each_entry(m, &mirred_list, tcfm_list) { - if (rcu_access_pointer(m->tcfm_dev) == dev) { + spin_lock(&m->tcf_lock); + if (tcf_mirred_dev_dereference(m) == dev) { dev_put(dev); /* Note : no rcu grace period necessary, as * net_device are already rcu protected. */ RCU_INIT_POINTER(m->tcfm_dev, NULL); } + spin_unlock(&m->tcf_lock); } + spin_unlock(&mirred_list_lock); } return NOTIFY_DONE; @@ -358,8 +379,20 @@ static struct notifier_block mirred_device_notifier = { static struct net_device *tcf_mirred_get_dev(const struct tc_action *a) { struct tcf_mirred *m = to_mirred(a); + struct net_device *dev; + + rcu_read_lock(); + dev = rcu_dereference(m->tcfm_dev); + if (dev) + dev_hold(dev); + rcu_read_unlock(); - return rtnl_dereference(m->tcfm_dev); + return dev; +} + +static void tcf_mirred_put_dev(struct net_device *dev) +{ + dev_put(dev); } static int tcf_mirred_delete(struct net *net, u32 index) @@ -373,7 +406,7 @@ static struct tc_action_ops act_mirred_ops = { .kind = "mirred", .type = TCA_ACT_MIRRED, .owner = THIS_MODULE, - .act = tcf_mirred, + .act = tcf_mirred_act, .stats_update = tcf_stats_update, .dump = tcf_mirred_dump, .cleanup = tcf_mirred_release, @@ -382,6 +415,7 @@ static struct tc_action_ops act_mirred_ops = { .lookup = tcf_mirred_search, .size = sizeof(struct tcf_mirred), .get_dev = tcf_mirred_get_dev, + .put_dev = tcf_mirred_put_dev, .delete = tcf_mirred_delete, }; diff --git a/net/sched/act_nat.c b/net/sched/act_nat.c index 4dd9188a72fd..822e903bfc25 100644 --- a/net/sched/act_nat.c +++ b/net/sched/act_nat.c @@ -93,8 +93,8 @@ static int tcf_nat_init(struct net *net, struct nlattr *nla, struct nlattr *est, return ret; } -static int tcf_nat(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_nat_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_nat *p = to_tcf_nat(a); struct iphdr *iph; @@ -311,7 +311,7 @@ static struct tc_action_ops act_nat_ops = { .kind = "nat", .type = TCA_ACT_NAT, .owner = THIS_MODULE, - .act = tcf_nat, + .act = tcf_nat_act, .dump = tcf_nat_dump, .init = tcf_nat_init, .walk = tcf_nat_walker, diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index 43ba999b2d23..8a7a7cb94e83 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -187,44 +187,38 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla, tcf_idr_cleanup(tn, parm->index); goto out_free; } - p = to_pedit(*a); - keys = kmalloc(ksize, GFP_KERNEL); - if (!keys) { - tcf_idr_release(*a, bind); - ret = -ENOMEM; - goto out_free; - } ret = ACT_P_CREATED; } else if (err > 0) { if (bind) goto out_free; if (!ovr) { - tcf_idr_release(*a, bind); ret = -EEXIST; - goto out_free; - } - p = to_pedit(*a); - if (p->tcfp_nkeys && p->tcfp_nkeys != parm->nkeys) { - keys = kmalloc(ksize, GFP_KERNEL); - if (!keys) { - ret = -ENOMEM; - goto out_free; - } + goto out_release; } } else { return err; } + p = to_pedit(*a); spin_lock_bh(&p->tcf_lock); - p->tcfp_flags = parm->flags; - p->tcf_action = parm->action; - if (keys) { + + if (ret == ACT_P_CREATED || + (p->tcfp_nkeys && p->tcfp_nkeys != parm->nkeys)) { + keys = kmalloc(ksize, GFP_ATOMIC); + if (!keys) { + spin_unlock_bh(&p->tcf_lock); + ret = -ENOMEM; + goto out_release; + } kfree(p->tcfp_keys); p->tcfp_keys = keys; p->tcfp_nkeys = parm->nkeys; } memcpy(p->tcfp_keys, parm->keys, ksize); + p->tcfp_flags = parm->flags; + p->tcf_action = parm->action; + kfree(p->tcfp_keys_ex); p->tcfp_keys_ex = keys_ex; @@ -232,6 +226,9 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla, if (ret == ACT_P_CREATED) tcf_idr_insert(tn, *a); return ret; + +out_release: + tcf_idr_release(*a, bind); out_free: kfree(keys_ex); return ret; @@ -291,8 +288,8 @@ static int pedit_skb_hdr_offset(struct sk_buff *skb, return ret; } -static int tcf_pedit(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_pedit_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_pedit *p = to_pedit(a); int i; @@ -410,6 +407,7 @@ static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, if (unlikely(!opt)) return -ENOBUFS; + spin_lock_bh(&p->tcf_lock); memcpy(opt->keys, p->tcfp_keys, p->tcfp_nkeys * sizeof(struct tc_pedit_key)); opt->index = p->tcf_index; @@ -432,11 +430,13 @@ static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &p->tcf_tm); if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD)) goto nla_put_failure; + spin_unlock_bh(&p->tcf_lock); kfree(opt); return skb->len; nla_put_failure: + spin_unlock_bh(&p->tcf_lock); nlmsg_trim(skb, b); kfree(opt); return -1; @@ -471,7 +471,7 @@ static struct tc_action_ops act_pedit_ops = { .kind = "pedit", .type = TCA_ACT_PEDIT, .owner = THIS_MODULE, - .act = tcf_pedit, + .act = tcf_pedit_act, .dump = tcf_pedit_dump, .cleanup = tcf_pedit_cleanup, .init = tcf_pedit_init, diff --git a/net/sched/act_police.c b/net/sched/act_police.c index 1f3192ea8df7..06f0742db593 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -56,7 +56,7 @@ struct tc_police_compat { static unsigned int police_net_id; static struct tc_action_ops act_police_ops; -static int tcf_act_police_walker(struct net *net, struct sk_buff *skb, +static int tcf_police_walker(struct net *net, struct sk_buff *skb, struct netlink_callback *cb, int type, const struct tc_action_ops *ops, struct netlink_ext_ack *extack) @@ -73,7 +73,7 @@ static const struct nla_policy police_policy[TCA_POLICE_MAX + 1] = { [TCA_POLICE_RESULT] = { .type = NLA_U32 }, }; -static int tcf_act_police_init(struct net *net, struct nlattr *nla, +static int tcf_police_init(struct net *net, struct nlattr *nla, struct nlattr *est, struct tc_action **a, int ovr, int bind, bool rtnl_held, struct netlink_ext_ack *extack) @@ -203,7 +203,7 @@ failure: return err; } -static int tcf_act_police(struct sk_buff *skb, const struct tc_action *a, +static int tcf_police_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) { struct tcf_police *police = to_police(a); @@ -267,21 +267,22 @@ static int tcf_act_police(struct sk_buff *skb, const struct tc_action *a, return police->tcf_action; } -static int tcf_act_police_dump(struct sk_buff *skb, struct tc_action *a, +static int tcf_police_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { unsigned char *b = skb_tail_pointer(skb); struct tcf_police *police = to_police(a); struct tc_police opt = { .index = police->tcf_index, - .action = police->tcf_action, - .mtu = police->tcfp_mtu, - .burst = PSCHED_NS2TICKS(police->tcfp_burst), .refcnt = refcount_read(&police->tcf_refcnt) - ref, .bindcnt = atomic_read(&police->tcf_bindcnt) - bind, }; struct tcf_t t; + spin_lock_bh(&police->tcf_lock); + opt.action = police->tcf_action; + opt.mtu = police->tcfp_mtu; + opt.burst = PSCHED_NS2TICKS(police->tcfp_burst); if (police->rate_present) psched_ratecfg_getrate(&opt.rate, &police->rate); if (police->peak_present) @@ -301,10 +302,12 @@ static int tcf_act_police_dump(struct sk_buff *skb, struct tc_action *a, t.expires = jiffies_to_clock_t(police->tcf_tm.expires); if (nla_put_64bit(skb, TCA_POLICE_TM, sizeof(t), &t, TCA_POLICE_PAD)) goto nla_put_failure; + spin_unlock_bh(&police->tcf_lock); return skb->len; nla_put_failure: + spin_unlock_bh(&police->tcf_lock); nlmsg_trim(skb, b); return -1; } @@ -332,10 +335,10 @@ static struct tc_action_ops act_police_ops = { .kind = "police", .type = TCA_ID_POLICE, .owner = THIS_MODULE, - .act = tcf_act_police, - .dump = tcf_act_police_dump, - .init = tcf_act_police_init, - .walk = tcf_act_police_walker, + .act = tcf_police_act, + .dump = tcf_police_dump, + .init = tcf_police_init, + .walk = tcf_police_walker, .lookup = tcf_police_search, .delete = tcf_police_delete, .size = sizeof(struct tcf_police), diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 2608ccc83e5e..81071afe1b43 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -80,11 +80,13 @@ static int tcf_sample_init(struct net *net, struct nlattr *nla, } s = to_sample(*a); + spin_lock(&s->tcf_lock); s->tcf_action = parm->action; s->rate = nla_get_u32(tb[TCA_SAMPLE_RATE]); s->psample_group_num = nla_get_u32(tb[TCA_SAMPLE_PSAMPLE_GROUP]); psample_group = psample_group_get(net, s->psample_group_num); if (!psample_group) { + spin_unlock(&s->tcf_lock); tcf_idr_release(*a, bind); return -ENOMEM; } @@ -94,6 +96,7 @@ static int tcf_sample_init(struct net *net, struct nlattr *nla, s->truncate = true; s->trunc_size = nla_get_u32(tb[TCA_SAMPLE_TRUNC_SIZE]); } + spin_unlock(&s->tcf_lock); if (ret == ACT_P_CREATED) tcf_idr_insert(tn, *a); @@ -105,7 +108,8 @@ static void tcf_sample_cleanup(struct tc_action *a) struct tcf_sample *s = to_sample(a); struct psample_group *psample_group; - psample_group = rtnl_dereference(s->psample_group); + /* last reference to action, no need to lock */ + psample_group = rcu_dereference_protected(s->psample_group, 1); RCU_INIT_POINTER(s->psample_group, NULL); if (psample_group) psample_group_put(psample_group); @@ -174,12 +178,13 @@ static int tcf_sample_dump(struct sk_buff *skb, struct tc_action *a, struct tcf_sample *s = to_sample(a); struct tc_sample opt = { .index = s->tcf_index, - .action = s->tcf_action, .refcnt = refcount_read(&s->tcf_refcnt) - ref, .bindcnt = atomic_read(&s->tcf_bindcnt) - bind, }; struct tcf_t t; + spin_lock(&s->tcf_lock); + opt.action = s->tcf_action; if (nla_put(skb, TCA_SAMPLE_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -196,9 +201,12 @@ static int tcf_sample_dump(struct sk_buff *skb, struct tc_action *a, if (nla_put_u32(skb, TCA_SAMPLE_PSAMPLE_GROUP, s->psample_group_num)) goto nla_put_failure; + spin_unlock(&s->tcf_lock); + return skb->len; nla_put_failure: + spin_unlock(&s->tcf_lock); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_simple.c b/net/sched/act_simple.c index aa51152e0066..e616523ba3c1 100644 --- a/net/sched/act_simple.c +++ b/net/sched/act_simple.c @@ -28,8 +28,8 @@ static unsigned int simp_net_id; static struct tc_action_ops act_simp_ops; #define SIMP_MAX_DATA 32 -static int tcf_simp(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_simp_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_defact *d = to_defact(a); @@ -156,10 +156,11 @@ static int tcf_simp_dump(struct sk_buff *skb, struct tc_action *a, .index = d->tcf_index, .refcnt = refcount_read(&d->tcf_refcnt) - ref, .bindcnt = atomic_read(&d->tcf_bindcnt) - bind, - .action = d->tcf_action, }; struct tcf_t t; + spin_lock_bh(&d->tcf_lock); + opt.action = d->tcf_action; if (nla_put(skb, TCA_DEF_PARMS, sizeof(opt), &opt) || nla_put_string(skb, TCA_DEF_DATA, d->tcfd_defdata)) goto nla_put_failure; @@ -167,9 +168,12 @@ static int tcf_simp_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &d->tcf_tm); if (nla_put_64bit(skb, TCA_DEF_TM, sizeof(t), &t, TCA_DEF_PAD)) goto nla_put_failure; + spin_unlock_bh(&d->tcf_lock); + return skb->len; nla_put_failure: + spin_unlock_bh(&d->tcf_lock); nlmsg_trim(skb, b); return -1; } @@ -203,7 +207,7 @@ static struct tc_action_ops act_simp_ops = { .kind = "simple", .type = TCA_ACT_SIMP, .owner = THIS_MODULE, - .act = tcf_simp, + .act = tcf_simp_act, .dump = tcf_simp_dump, .cleanup = tcf_simp_release, .init = tcf_simp_init, diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index a6db47ebec11..926d7bc4a89d 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -33,8 +33,8 @@ static unsigned int skbedit_net_id; static struct tc_action_ops act_skbedit_ops; -static int tcf_skbedit(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_skbedit_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_skbedit *d = to_skbedit(a); struct tcf_skbedit_params *params; @@ -310,7 +310,7 @@ static struct tc_action_ops act_skbedit_ops = { .kind = "skbedit", .type = TCA_ACT_SKBEDIT, .owner = THIS_MODULE, - .act = tcf_skbedit, + .act = tcf_skbedit_act, .dump = tcf_skbedit_dump, .init = tcf_skbedit_init, .cleanup = tcf_skbedit_cleanup, diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index c437c6d51a71..d6a1af0c4171 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -24,7 +24,7 @@ static unsigned int skbmod_net_id; static struct tc_action_ops act_skbmod_ops; #define MAX_EDIT_LEN ETH_HLEN -static int tcf_skbmod_run(struct sk_buff *skb, const struct tc_action *a, +static int tcf_skbmod_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) { struct tcf_skbmod *d = to_skbmod(a); @@ -156,7 +156,6 @@ static int tcf_skbmod_init(struct net *net, struct nlattr *nla, d = to_skbmod(*a); - ASSERT_RTNL(); p = kzalloc(sizeof(struct tcf_skbmod_params), GFP_KERNEL); if (unlikely(!p)) { tcf_idr_release(*a, bind); @@ -166,10 +165,10 @@ static int tcf_skbmod_init(struct net *net, struct nlattr *nla, p->flags = lflags; d->tcf_action = parm->action; - p_old = rtnl_dereference(d->skbmod_p); - if (ovr) spin_lock_bh(&d->tcf_lock); + /* Protected by tcf_lock if overwriting existing action. */ + p_old = rcu_dereference_protected(d->skbmod_p, 1); if (lflags & SKBMOD_F_DMAC) ether_addr_copy(p->eth_dst, daddr); @@ -205,15 +204,18 @@ static int tcf_skbmod_dump(struct sk_buff *skb, struct tc_action *a, { struct tcf_skbmod *d = to_skbmod(a); unsigned char *b = skb_tail_pointer(skb); - struct tcf_skbmod_params *p = rtnl_dereference(d->skbmod_p); + struct tcf_skbmod_params *p; struct tc_skbmod opt = { .index = d->tcf_index, .refcnt = refcount_read(&d->tcf_refcnt) - ref, .bindcnt = atomic_read(&d->tcf_bindcnt) - bind, - .action = d->tcf_action, }; struct tcf_t t; + spin_lock_bh(&d->tcf_lock); + opt.action = d->tcf_action; + p = rcu_dereference_protected(d->skbmod_p, + lockdep_is_held(&d->tcf_lock)); opt.flags = p->flags; if (nla_put(skb, TCA_SKBMOD_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -231,8 +233,10 @@ static int tcf_skbmod_dump(struct sk_buff *skb, struct tc_action *a, if (nla_put_64bit(skb, TCA_SKBMOD_TM, sizeof(t), &t, TCA_SKBMOD_PAD)) goto nla_put_failure; + spin_unlock_bh(&d->tcf_lock); return skb->len; nla_put_failure: + spin_unlock_bh(&d->tcf_lock); nlmsg_trim(skb, b); return -1; } @@ -266,7 +270,7 @@ static struct tc_action_ops act_skbmod_ops = { .kind = "skbmod", .type = TCA_ACT_SKBMOD, .owner = THIS_MODULE, - .act = tcf_skbmod_run, + .act = tcf_skbmod_act, .dump = tcf_skbmod_dump, .init = tcf_skbmod_init, .cleanup = tcf_skbmod_cleanup, diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index d42d9e112789..ba2ae9f75ef5 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -204,7 +204,6 @@ static int tunnel_key_init(struct net *net, struct nlattr *nla, { struct tc_action_net *tn = net_generic(net, tunnel_key_net_id); struct nlattr *tb[TCA_TUNNEL_KEY_MAX + 1]; - struct tcf_tunnel_key_params *params_old; struct tcf_tunnel_key_params *params_new; struct metadata_dst *metadata = NULL; struct tc_tunnel_key *parm; @@ -346,24 +345,22 @@ static int tunnel_key_init(struct net *net, struct nlattr *nla, t = to_tunnel_key(*a); - ASSERT_RTNL(); params_new = kzalloc(sizeof(*params_new), GFP_KERNEL); if (unlikely(!params_new)) { tcf_idr_release(*a, bind); NL_SET_ERR_MSG(extack, "Cannot allocate tunnel key parameters"); return -ENOMEM; } - - params_old = rtnl_dereference(t->params); - - t->tcf_action = parm->action; params_new->tcft_action = parm->t_action; params_new->tcft_enc_metadata = metadata; - rcu_assign_pointer(t->params, params_new); - - if (params_old) - kfree_rcu(params_old, rcu); + spin_lock(&t->tcf_lock); + t->tcf_action = parm->action; + rcu_swap_protected(t->params, params_new, + lockdep_is_held(&t->tcf_lock)); + spin_unlock(&t->tcf_lock); + if (params_new) + kfree_rcu(params_new, rcu); if (ret == ACT_P_CREATED) tcf_idr_insert(tn, *a); @@ -485,12 +482,13 @@ static int tunnel_key_dump(struct sk_buff *skb, struct tc_action *a, .index = t->tcf_index, .refcnt = refcount_read(&t->tcf_refcnt) - ref, .bindcnt = atomic_read(&t->tcf_bindcnt) - bind, - .action = t->tcf_action, }; struct tcf_t tm; - params = rtnl_dereference(t->params); - + spin_lock(&t->tcf_lock); + params = rcu_dereference_protected(t->params, + lockdep_is_held(&t->tcf_lock)); + opt.action = t->tcf_action; opt.t_action = params->tcft_action; if (nla_put(skb, TCA_TUNNEL_KEY_PARMS, sizeof(opt), &opt)) @@ -522,10 +520,12 @@ static int tunnel_key_dump(struct sk_buff *skb, struct tc_action *a, if (nla_put_64bit(skb, TCA_TUNNEL_KEY_TM, sizeof(tm), &tm, TCA_TUNNEL_KEY_PAD)) goto nla_put_failure; + spin_unlock(&t->tcf_lock); return skb->len; nla_put_failure: + spin_unlock(&t->tcf_lock); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_vlan.c b/net/sched/act_vlan.c index 15a0ee214c9c..d1f5028384c9 100644 --- a/net/sched/act_vlan.c +++ b/net/sched/act_vlan.c @@ -22,8 +22,8 @@ static unsigned int vlan_net_id; static struct tc_action_ops act_vlan_ops; -static int tcf_vlan(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +static int tcf_vlan_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct tcf_vlan *v = to_vlan(a); struct tcf_vlan_params *p; @@ -109,7 +109,7 @@ static int tcf_vlan_init(struct net *net, struct nlattr *nla, { struct tc_action_net *tn = net_generic(net, vlan_net_id); struct nlattr *tb[TCA_VLAN_MAX + 1]; - struct tcf_vlan_params *p, *p_old; + struct tcf_vlan_params *p; struct tc_vlan *parm; struct tcf_vlan *v; int action; @@ -202,26 +202,24 @@ static int tcf_vlan_init(struct net *net, struct nlattr *nla, v = to_vlan(*a); - ASSERT_RTNL(); p = kzalloc(sizeof(*p), GFP_KERNEL); if (!p) { tcf_idr_release(*a, bind); return -ENOMEM; } - v->tcf_action = parm->action; - - p_old = rtnl_dereference(v->vlan_p); - p->tcfv_action = action; p->tcfv_push_vid = push_vid; p->tcfv_push_prio = push_prio; p->tcfv_push_proto = push_proto; - rcu_assign_pointer(v->vlan_p, p); + spin_lock(&v->tcf_lock); + v->tcf_action = parm->action; + rcu_swap_protected(v->vlan_p, p, lockdep_is_held(&v->tcf_lock)); + spin_unlock(&v->tcf_lock); - if (p_old) - kfree_rcu(p_old, rcu); + if (p) + kfree_rcu(p, rcu); if (ret == ACT_P_CREATED) tcf_idr_insert(tn, *a); @@ -243,16 +241,18 @@ static int tcf_vlan_dump(struct sk_buff *skb, struct tc_action *a, { unsigned char *b = skb_tail_pointer(skb); struct tcf_vlan *v = to_vlan(a); - struct tcf_vlan_params *p = rtnl_dereference(v->vlan_p); + struct tcf_vlan_params *p; struct tc_vlan opt = { .index = v->tcf_index, .refcnt = refcount_read(&v->tcf_refcnt) - ref, .bindcnt = atomic_read(&v->tcf_bindcnt) - bind, - .action = v->tcf_action, - .v_action = p->tcfv_action, }; struct tcf_t t; + spin_lock(&v->tcf_lock); + opt.action = v->tcf_action; + p = rcu_dereference_protected(v->vlan_p, lockdep_is_held(&v->tcf_lock)); + opt.v_action = p->tcfv_action; if (nla_put(skb, TCA_VLAN_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -268,9 +268,12 @@ static int tcf_vlan_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &v->tcf_tm); if (nla_put_64bit(skb, TCA_VLAN_TM, sizeof(t), &t, TCA_VLAN_PAD)) goto nla_put_failure; + spin_unlock(&v->tcf_lock); + return skb->len; nla_put_failure: + spin_unlock(&v->tcf_lock); nlmsg_trim(skb, b); return -1; } @@ -304,7 +307,7 @@ static struct tc_action_ops act_vlan_ops = { .kind = "vlan", .type = TCA_ACT_VLAN, .owner = THIS_MODULE, - .act = tcf_vlan, + .act = tcf_vlan_act, .dump = tcf_vlan_dump, .init = tcf_vlan_init, .cleanup = tcf_vlan_cleanup, diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index f922ce27ed5e..31bd1439cf60 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2176,6 +2176,7 @@ static int tc_exts_setup_cb_egdev_call(struct tcf_exts *exts, if (!dev) continue; ret = tc_setup_cb_egdev_call(dev, type, type_data, err_stop); + a->ops->put_dev(dev); if (ret < 0) return ret; ok_count += ret; diff --git a/net/sctp/chunk.c b/net/sctp/chunk.c index bfb9f812e2ef..ce8087846f05 100644 --- a/net/sctp/chunk.c +++ b/net/sctp/chunk.c @@ -325,7 +325,8 @@ int sctp_chunk_abandoned(struct sctp_chunk *chunk) if (SCTP_PR_TTL_ENABLED(chunk->sinfo.sinfo_flags) && time_after(jiffies, chunk->msg->expires_at)) { struct sctp_stream_out *streamout = - &chunk->asoc->stream.out[chunk->sinfo.sinfo_stream]; + SCTP_SO(&chunk->asoc->stream, + chunk->sinfo.sinfo_stream); if (chunk->sent_count) { chunk->asoc->abandoned_sent[SCTP_PR_INDEX(TTL)]++; @@ -339,7 +340,8 @@ int sctp_chunk_abandoned(struct sctp_chunk *chunk) } else if (SCTP_PR_RTX_ENABLED(chunk->sinfo.sinfo_flags) && chunk->sent_count > chunk->sinfo.sinfo_timetolive) { struct sctp_stream_out *streamout = - &chunk->asoc->stream.out[chunk->sinfo.sinfo_stream]; + SCTP_SO(&chunk->asoc->stream, + chunk->sinfo.sinfo_stream); chunk->asoc->abandoned_sent[SCTP_PR_INDEX(RTX)]++; streamout->ext->abandoned_sent[SCTP_PR_INDEX(RTX)]++; diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c index d68aa33485a9..d74d00b29942 100644 --- a/net/sctp/outqueue.c +++ b/net/sctp/outqueue.c @@ -80,7 +80,7 @@ static inline void sctp_outq_head_data(struct sctp_outq *q, q->out_qlen += ch->skb->len; stream = sctp_chunk_stream_no(ch); - oute = q->asoc->stream.out[stream].ext; + oute = SCTP_SO(&q->asoc->stream, stream)->ext; list_add(&ch->stream_list, &oute->outq); } @@ -101,7 +101,7 @@ static inline void sctp_outq_tail_data(struct sctp_outq *q, q->out_qlen += ch->skb->len; stream = sctp_chunk_stream_no(ch); - oute = q->asoc->stream.out[stream].ext; + oute = SCTP_SO(&q->asoc->stream, stream)->ext; list_add_tail(&ch->stream_list, &oute->outq); } @@ -372,7 +372,7 @@ static int sctp_prsctp_prune_sent(struct sctp_association *asoc, sctp_insert_list(&asoc->outqueue.abandoned, &chk->transmitted_list); - streamout = &asoc->stream.out[chk->sinfo.sinfo_stream]; + streamout = SCTP_SO(&asoc->stream, chk->sinfo.sinfo_stream); asoc->sent_cnt_removable--; asoc->abandoned_sent[SCTP_PR_INDEX(PRIO)]++; streamout->ext->abandoned_sent[SCTP_PR_INDEX(PRIO)]++; @@ -416,7 +416,7 @@ static int sctp_prsctp_prune_unsent(struct sctp_association *asoc, asoc->abandoned_unsent[SCTP_PR_INDEX(PRIO)]++; if (chk->sinfo.sinfo_stream < asoc->stream.outcnt) { struct sctp_stream_out *streamout = - &asoc->stream.out[chk->sinfo.sinfo_stream]; + SCTP_SO(&asoc->stream, chk->sinfo.sinfo_stream); streamout->ext->abandoned_unsent[SCTP_PR_INDEX(PRIO)]++; } @@ -1082,6 +1082,7 @@ static void sctp_outq_flush_data(struct sctp_flush_ctx *ctx, /* Finally, transmit new packets. */ while ((chunk = sctp_outq_dequeue_data(ctx->q)) != NULL) { __u32 sid = ntohs(chunk->subh.data_hdr->stream); + __u8 stream_state = SCTP_SO(&ctx->asoc->stream, sid)->state; /* Has this chunk expired? */ if (sctp_chunk_abandoned(chunk)) { @@ -1091,7 +1092,7 @@ static void sctp_outq_flush_data(struct sctp_flush_ctx *ctx, continue; } - if (ctx->asoc->stream.out[sid].state == SCTP_STREAM_CLOSED) { + if (stream_state == SCTP_STREAM_CLOSED) { sctp_outq_head_data(ctx->q, chunk); break; } diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 502c0d7cb105..e96b15a66aba 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -1911,7 +1911,7 @@ static int sctp_sendmsg_to_asoc(struct sctp_association *asoc, goto err; } - if (unlikely(!asoc->stream.out[sinfo->sinfo_stream].ext)) { + if (unlikely(!SCTP_SO(&asoc->stream, sinfo->sinfo_stream)->ext)) { err = sctp_stream_init_ext(&asoc->stream, sinfo->sinfo_stream); if (err) goto err; @@ -7154,7 +7154,7 @@ static int sctp_getsockopt_pr_streamstatus(struct sock *sk, int len, if (!asoc || params.sprstat_sid >= asoc->stream.outcnt) goto out; - streamoute = asoc->stream.out[params.sprstat_sid].ext; + streamoute = SCTP_SO(&asoc->stream, params.sprstat_sid)->ext; if (!streamoute) { /* Not allocated yet, means all stats are 0 */ params.sprstat_abandoned_unsent = 0; diff --git a/net/sctp/stream.c b/net/sctp/stream.c index f1f1d1b232ba..ffb940d3b57c 100644 --- a/net/sctp/stream.c +++ b/net/sctp/stream.c @@ -37,6 +37,53 @@ #include <net/sctp/sm.h> #include <net/sctp/stream_sched.h> +static struct flex_array *fa_alloc(size_t elem_size, size_t elem_count, + gfp_t gfp) +{ + struct flex_array *result; + int err; + + result = flex_array_alloc(elem_size, elem_count, gfp); + if (result) { + err = flex_array_prealloc(result, 0, elem_count, gfp); + if (err) { + flex_array_free(result); + result = NULL; + } + } + + return result; +} + +static void fa_free(struct flex_array *fa) +{ + if (fa) + flex_array_free(fa); +} + +static void fa_copy(struct flex_array *fa, struct flex_array *from, + size_t index, size_t count) +{ + void *elem; + + while (count--) { + elem = flex_array_get(from, index); + flex_array_put(fa, index, elem, 0); + index++; + } +} + +static void fa_zero(struct flex_array *fa, size_t index, size_t count) +{ + void *elem; + + while (count--) { + elem = flex_array_get(fa, index); + memset(elem, 0, fa->element_size); + index++; + } +} + /* Migrates chunks from stream queues to new stream queues if needed, * but not across associations. Also, removes those chunks to streams * higher than the new max. @@ -78,34 +125,33 @@ static void sctp_stream_outq_migrate(struct sctp_stream *stream, * sctp_stream_update will swap ->out pointers. */ for (i = 0; i < outcnt; i++) { - kfree(new->out[i].ext); - new->out[i].ext = stream->out[i].ext; - stream->out[i].ext = NULL; + kfree(SCTP_SO(new, i)->ext); + SCTP_SO(new, i)->ext = SCTP_SO(stream, i)->ext; + SCTP_SO(stream, i)->ext = NULL; } } for (i = outcnt; i < stream->outcnt; i++) - kfree(stream->out[i].ext); + kfree(SCTP_SO(stream, i)->ext); } static int sctp_stream_alloc_out(struct sctp_stream *stream, __u16 outcnt, gfp_t gfp) { - struct sctp_stream_out *out; + struct flex_array *out; + size_t elem_size = sizeof(struct sctp_stream_out); - out = kmalloc_array(outcnt, sizeof(*out), gfp); + out = fa_alloc(elem_size, outcnt, gfp); if (!out) return -ENOMEM; if (stream->out) { - memcpy(out, stream->out, min(outcnt, stream->outcnt) * - sizeof(*out)); - kfree(stream->out); + fa_copy(out, stream->out, 0, min(outcnt, stream->outcnt)); + fa_free(stream->out); } if (outcnt > stream->outcnt) - memset(out + stream->outcnt, 0, - (outcnt - stream->outcnt) * sizeof(*out)); + fa_zero(out, stream->outcnt, (outcnt - stream->outcnt)); stream->out = out; @@ -115,22 +161,20 @@ static int sctp_stream_alloc_out(struct sctp_stream *stream, __u16 outcnt, static int sctp_stream_alloc_in(struct sctp_stream *stream, __u16 incnt, gfp_t gfp) { - struct sctp_stream_in *in; - - in = kmalloc_array(incnt, sizeof(*stream->in), gfp); + struct flex_array *in; + size_t elem_size = sizeof(struct sctp_stream_in); + in = fa_alloc(elem_size, incnt, gfp); if (!in) return -ENOMEM; if (stream->in) { - memcpy(in, stream->in, min(incnt, stream->incnt) * - sizeof(*in)); - kfree(stream->in); + fa_copy(in, stream->in, 0, min(incnt, stream->incnt)); + fa_free(stream->in); } if (incnt > stream->incnt) - memset(in + stream->incnt, 0, - (incnt - stream->incnt) * sizeof(*in)); + fa_zero(in, stream->incnt, (incnt - stream->incnt)); stream->in = in; @@ -162,7 +206,7 @@ int sctp_stream_init(struct sctp_stream *stream, __u16 outcnt, __u16 incnt, stream->outcnt = outcnt; for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_OPEN; + SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; sched->init(stream); @@ -174,7 +218,7 @@ in: ret = sctp_stream_alloc_in(stream, incnt, gfp); if (ret) { sched->free(stream); - kfree(stream->out); + fa_free(stream->out); stream->out = NULL; stream->outcnt = 0; goto out; @@ -193,7 +237,7 @@ int sctp_stream_init_ext(struct sctp_stream *stream, __u16 sid) soute = kzalloc(sizeof(*soute), GFP_KERNEL); if (!soute) return -ENOMEM; - stream->out[sid].ext = soute; + SCTP_SO(stream, sid)->ext = soute; return sctp_sched_init_sid(stream, sid, GFP_KERNEL); } @@ -205,9 +249,9 @@ void sctp_stream_free(struct sctp_stream *stream) sched->free(stream); for (i = 0; i < stream->outcnt; i++) - kfree(stream->out[i].ext); - kfree(stream->out); - kfree(stream->in); + kfree(SCTP_SO(stream, i)->ext); + fa_free(stream->out); + fa_free(stream->in); } void sctp_stream_clear(struct sctp_stream *stream) @@ -215,12 +259,12 @@ void sctp_stream_clear(struct sctp_stream *stream) int i; for (i = 0; i < stream->outcnt; i++) { - stream->out[i].mid = 0; - stream->out[i].mid_uo = 0; + SCTP_SO(stream, i)->mid = 0; + SCTP_SO(stream, i)->mid_uo = 0; } for (i = 0; i < stream->incnt; i++) - stream->in[i].mid = 0; + SCTP_SI(stream, i)->mid = 0; } void sctp_stream_update(struct sctp_stream *stream, struct sctp_stream *new) @@ -273,8 +317,8 @@ static bool sctp_stream_outq_is_empty(struct sctp_stream *stream, for (i = 0; i < str_nums; i++) { __u16 sid = ntohs(str_list[i]); - if (stream->out[sid].ext && - !list_empty(&stream->out[sid].ext->outq)) + if (SCTP_SO(stream, sid)->ext && + !list_empty(&SCTP_SO(stream, sid)->ext->outq)) return false; } @@ -361,11 +405,11 @@ int sctp_send_reset_streams(struct sctp_association *asoc, if (out) { if (str_nums) for (i = 0; i < str_nums; i++) - stream->out[str_list[i]].state = + SCTP_SO(stream, str_list[i])->state = SCTP_STREAM_CLOSED; else for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_CLOSED; + SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED; } asoc->strreset_chunk = chunk; @@ -380,11 +424,11 @@ int sctp_send_reset_streams(struct sctp_association *asoc, if (str_nums) for (i = 0; i < str_nums; i++) - stream->out[str_list[i]].state = + SCTP_SO(stream, str_list[i])->state = SCTP_STREAM_OPEN; else for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_OPEN; + SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; goto out; } @@ -418,7 +462,7 @@ int sctp_send_reset_assoc(struct sctp_association *asoc) /* Block further xmit of data until this request is completed */ for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_CLOSED; + SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED; asoc->strreset_chunk = chunk; sctp_chunk_hold(asoc->strreset_chunk); @@ -429,7 +473,7 @@ int sctp_send_reset_assoc(struct sctp_association *asoc) asoc->strreset_chunk = NULL; for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_OPEN; + SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; return retval; } @@ -609,10 +653,10 @@ struct sctp_chunk *sctp_process_strreset_outreq( } for (i = 0; i < nums; i++) - stream->in[ntohs(str_p[i])].mid = 0; + SCTP_SI(stream, ntohs(str_p[i]))->mid = 0; } else { for (i = 0; i < stream->incnt; i++) - stream->in[i].mid = 0; + SCTP_SI(stream, i)->mid = 0; } result = SCTP_STRRESET_PERFORMED; @@ -683,11 +727,11 @@ struct sctp_chunk *sctp_process_strreset_inreq( if (nums) for (i = 0; i < nums; i++) - stream->out[ntohs(str_p[i])].state = + SCTP_SO(stream, ntohs(str_p[i]))->state = SCTP_STREAM_CLOSED; else for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_CLOSED; + SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED; asoc->strreset_chunk = chunk; asoc->strreset_outstanding = 1; @@ -786,11 +830,11 @@ struct sctp_chunk *sctp_process_strreset_tsnreq( * incoming and outgoing streams. */ for (i = 0; i < stream->outcnt; i++) { - stream->out[i].mid = 0; - stream->out[i].mid_uo = 0; + SCTP_SO(stream, i)->mid = 0; + SCTP_SO(stream, i)->mid_uo = 0; } for (i = 0; i < stream->incnt; i++) - stream->in[i].mid = 0; + SCTP_SI(stream, i)->mid = 0; result = SCTP_STRRESET_PERFORMED; @@ -979,15 +1023,18 @@ struct sctp_chunk *sctp_process_strreset_resp( sizeof(__u16); if (result == SCTP_STRRESET_PERFORMED) { + struct sctp_stream_out *sout; if (nums) { for (i = 0; i < nums; i++) { - stream->out[ntohs(str_p[i])].mid = 0; - stream->out[ntohs(str_p[i])].mid_uo = 0; + sout = SCTP_SO(stream, ntohs(str_p[i])); + sout->mid = 0; + sout->mid_uo = 0; } } else { for (i = 0; i < stream->outcnt; i++) { - stream->out[i].mid = 0; - stream->out[i].mid_uo = 0; + sout = SCTP_SO(stream, i); + sout->mid = 0; + sout->mid_uo = 0; } } @@ -995,7 +1042,7 @@ struct sctp_chunk *sctp_process_strreset_resp( } for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_OPEN; + SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; *evp = sctp_ulpevent_make_stream_reset_event(asoc, flags, nums, str_p, GFP_ATOMIC); @@ -1050,15 +1097,15 @@ struct sctp_chunk *sctp_process_strreset_resp( asoc->adv_peer_ack_point = asoc->ctsn_ack_point; for (i = 0; i < stream->outcnt; i++) { - stream->out[i].mid = 0; - stream->out[i].mid_uo = 0; + SCTP_SO(stream, i)->mid = 0; + SCTP_SO(stream, i)->mid_uo = 0; } for (i = 0; i < stream->incnt; i++) - stream->in[i].mid = 0; + SCTP_SI(stream, i)->mid = 0; } for (i = 0; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_OPEN; + SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; *evp = sctp_ulpevent_make_assoc_reset_event(asoc, flags, stsn, rtsn, GFP_ATOMIC); @@ -1072,7 +1119,7 @@ struct sctp_chunk *sctp_process_strreset_resp( if (result == SCTP_STRRESET_PERFORMED) for (i = number; i < stream->outcnt; i++) - stream->out[i].state = SCTP_STREAM_OPEN; + SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; else stream->outcnt = number; diff --git a/net/sctp/stream_interleave.c b/net/sctp/stream_interleave.c index d3764c181299..0a78cdf86463 100644 --- a/net/sctp/stream_interleave.c +++ b/net/sctp/stream_interleave.c @@ -197,7 +197,7 @@ static struct sctp_ulpevent *sctp_intl_retrieve_partial( __u32 next_fsn = 0; int is_last = 0; - sin = sctp_stream_in(ulpq->asoc, event->stream); + sin = sctp_stream_in(&ulpq->asoc->stream, event->stream); skb_queue_walk(&ulpq->reasm, pos) { struct sctp_ulpevent *cevent = sctp_skb2event(pos); @@ -278,7 +278,7 @@ static struct sctp_ulpevent *sctp_intl_retrieve_reassembled( __u32 pd_len = 0; __u32 mid = 0; - sin = sctp_stream_in(ulpq->asoc, event->stream); + sin = sctp_stream_in(&ulpq->asoc->stream, event->stream); skb_queue_walk(&ulpq->reasm, pos) { struct sctp_ulpevent *cevent = sctp_skb2event(pos); @@ -368,7 +368,7 @@ static struct sctp_ulpevent *sctp_intl_reasm(struct sctp_ulpq *ulpq, sctp_intl_store_reasm(ulpq, event); - sin = sctp_stream_in(ulpq->asoc, event->stream); + sin = sctp_stream_in(&ulpq->asoc->stream, event->stream); if (sin->pd_mode && event->mid == sin->mid && event->fsn == sin->fsn) retval = sctp_intl_retrieve_partial(ulpq, event); @@ -575,7 +575,7 @@ static struct sctp_ulpevent *sctp_intl_retrieve_partial_uo( __u32 next_fsn = 0; int is_last = 0; - sin = sctp_stream_in(ulpq->asoc, event->stream); + sin = sctp_stream_in(&ulpq->asoc->stream, event->stream); skb_queue_walk(&ulpq->reasm_uo, pos) { struct sctp_ulpevent *cevent = sctp_skb2event(pos); @@ -659,7 +659,7 @@ static struct sctp_ulpevent *sctp_intl_retrieve_reassembled_uo( __u32 pd_len = 0; __u32 mid = 0; - sin = sctp_stream_in(ulpq->asoc, event->stream); + sin = sctp_stream_in(&ulpq->asoc->stream, event->stream); skb_queue_walk(&ulpq->reasm_uo, pos) { struct sctp_ulpevent *cevent = sctp_skb2event(pos); @@ -750,7 +750,7 @@ static struct sctp_ulpevent *sctp_intl_reasm_uo(struct sctp_ulpq *ulpq, sctp_intl_store_reasm_uo(ulpq, event); - sin = sctp_stream_in(ulpq->asoc, event->stream); + sin = sctp_stream_in(&ulpq->asoc->stream, event->stream); if (sin->pd_mode_uo && event->mid == sin->mid_uo && event->fsn == sin->fsn_uo) retval = sctp_intl_retrieve_partial_uo(ulpq, event); @@ -774,7 +774,7 @@ static struct sctp_ulpevent *sctp_intl_retrieve_first_uo(struct sctp_ulpq *ulpq) skb_queue_walk(&ulpq->reasm_uo, pos) { struct sctp_ulpevent *cevent = sctp_skb2event(pos); - csin = sctp_stream_in(ulpq->asoc, cevent->stream); + csin = sctp_stream_in(&ulpq->asoc->stream, cevent->stream); if (csin->pd_mode_uo) continue; @@ -875,7 +875,7 @@ static struct sctp_ulpevent *sctp_intl_retrieve_first(struct sctp_ulpq *ulpq) skb_queue_walk(&ulpq->reasm, pos) { struct sctp_ulpevent *cevent = sctp_skb2event(pos); - csin = sctp_stream_in(ulpq->asoc, cevent->stream); + csin = sctp_stream_in(&ulpq->asoc->stream, cevent->stream); if (csin->pd_mode) continue; @@ -1053,7 +1053,7 @@ static void sctp_intl_abort_pd(struct sctp_ulpq *ulpq, gfp_t gfp) __u16 sid; for (sid = 0; sid < stream->incnt; sid++) { - struct sctp_stream_in *sin = &stream->in[sid]; + struct sctp_stream_in *sin = SCTP_SI(stream, sid); __u32 mid; if (sin->pd_mode_uo) { @@ -1247,7 +1247,7 @@ static void sctp_handle_fwdtsn(struct sctp_ulpq *ulpq, struct sctp_chunk *chunk) static void sctp_intl_skip(struct sctp_ulpq *ulpq, __u16 sid, __u32 mid, __u8 flags) { - struct sctp_stream_in *sin = sctp_stream_in(ulpq->asoc, sid); + struct sctp_stream_in *sin = sctp_stream_in(&ulpq->asoc->stream, sid); struct sctp_stream *stream = &ulpq->asoc->stream; if (flags & SCTP_FTSN_U_BIT) { diff --git a/net/sctp/stream_sched.c b/net/sctp/stream_sched.c index f5fcd425232a..a6c04a94b08f 100644 --- a/net/sctp/stream_sched.c +++ b/net/sctp/stream_sched.c @@ -161,7 +161,7 @@ int sctp_sched_set_sched(struct sctp_association *asoc, /* Give the next scheduler a clean slate. */ for (i = 0; i < asoc->stream.outcnt; i++) { - void *p = asoc->stream.out[i].ext; + void *p = SCTP_SO(&asoc->stream, i)->ext; if (!p) continue; @@ -175,7 +175,7 @@ int sctp_sched_set_sched(struct sctp_association *asoc, asoc->outqueue.sched = n; n->init(&asoc->stream); for (i = 0; i < asoc->stream.outcnt; i++) { - if (!asoc->stream.out[i].ext) + if (!SCTP_SO(&asoc->stream, i)->ext) continue; ret = n->init_sid(&asoc->stream, i, GFP_KERNEL); @@ -217,7 +217,7 @@ int sctp_sched_set_value(struct sctp_association *asoc, __u16 sid, if (sid >= asoc->stream.outcnt) return -EINVAL; - if (!asoc->stream.out[sid].ext) { + if (!SCTP_SO(&asoc->stream, sid)->ext) { int ret; ret = sctp_stream_init_ext(&asoc->stream, sid); @@ -234,7 +234,7 @@ int sctp_sched_get_value(struct sctp_association *asoc, __u16 sid, if (sid >= asoc->stream.outcnt) return -EINVAL; - if (!asoc->stream.out[sid].ext) + if (!SCTP_SO(&asoc->stream, sid)->ext) return 0; return asoc->outqueue.sched->get(&asoc->stream, sid, value); @@ -252,7 +252,7 @@ void sctp_sched_dequeue_done(struct sctp_outq *q, struct sctp_chunk *ch) * priority stream comes in. */ sid = sctp_chunk_stream_no(ch); - sout = &q->asoc->stream.out[sid]; + sout = SCTP_SO(&q->asoc->stream, sid); q->asoc->stream.out_curr = sout; return; } @@ -272,8 +272,9 @@ void sctp_sched_dequeue_common(struct sctp_outq *q, struct sctp_chunk *ch) int sctp_sched_init_sid(struct sctp_stream *stream, __u16 sid, gfp_t gfp) { struct sctp_sched_ops *sched = sctp_sched_ops_from_stream(stream); + struct sctp_stream_out_ext *ext = SCTP_SO(stream, sid)->ext; - INIT_LIST_HEAD(&stream->out[sid].ext->outq); + INIT_LIST_HEAD(&ext->outq); return sched->init_sid(stream, sid, gfp); } diff --git a/net/sctp/stream_sched_prio.c b/net/sctp/stream_sched_prio.c index 7997d35dd0fd..2245083a98f2 100644 --- a/net/sctp/stream_sched_prio.c +++ b/net/sctp/stream_sched_prio.c @@ -75,10 +75,10 @@ static struct sctp_stream_priorities *sctp_sched_prio_get_head( /* No luck. So we search on all streams now. */ for (i = 0; i < stream->outcnt; i++) { - if (!stream->out[i].ext) + if (!SCTP_SO(stream, i)->ext) continue; - p = stream->out[i].ext->prio_head; + p = SCTP_SO(stream, i)->ext->prio_head; if (!p) /* Means all other streams won't be initialized * as well. @@ -165,7 +165,7 @@ static void sctp_sched_prio_sched(struct sctp_stream *stream, static int sctp_sched_prio_set(struct sctp_stream *stream, __u16 sid, __u16 prio, gfp_t gfp) { - struct sctp_stream_out *sout = &stream->out[sid]; + struct sctp_stream_out *sout = SCTP_SO(stream, sid); struct sctp_stream_out_ext *soute = sout->ext; struct sctp_stream_priorities *prio_head, *old; bool reschedule = false; @@ -186,7 +186,7 @@ static int sctp_sched_prio_set(struct sctp_stream *stream, __u16 sid, return 0; for (i = 0; i < stream->outcnt; i++) { - soute = stream->out[i].ext; + soute = SCTP_SO(stream, i)->ext; if (soute && soute->prio_head == old) /* It's still in use, nothing else to do here. */ return 0; @@ -201,7 +201,7 @@ static int sctp_sched_prio_set(struct sctp_stream *stream, __u16 sid, static int sctp_sched_prio_get(struct sctp_stream *stream, __u16 sid, __u16 *value) { - *value = stream->out[sid].ext->prio_head->prio; + *value = SCTP_SO(stream, sid)->ext->prio_head->prio; return 0; } @@ -215,7 +215,7 @@ static int sctp_sched_prio_init(struct sctp_stream *stream) static int sctp_sched_prio_init_sid(struct sctp_stream *stream, __u16 sid, gfp_t gfp) { - INIT_LIST_HEAD(&stream->out[sid].ext->prio_list); + INIT_LIST_HEAD(&SCTP_SO(stream, sid)->ext->prio_list); return sctp_sched_prio_set(stream, sid, 0, gfp); } @@ -233,9 +233,9 @@ static void sctp_sched_prio_free(struct sctp_stream *stream) */ sctp_sched_prio_unsched_all(stream); for (i = 0; i < stream->outcnt; i++) { - if (!stream->out[i].ext) + if (!SCTP_SO(stream, i)->ext) continue; - prio = stream->out[i].ext->prio_head; + prio = SCTP_SO(stream, i)->ext->prio_head; if (prio && list_empty(&prio->prio_sched)) list_add(&prio->prio_sched, &list); } @@ -255,7 +255,7 @@ static void sctp_sched_prio_enqueue(struct sctp_outq *q, ch = list_first_entry(&msg->chunks, struct sctp_chunk, frag_list); sid = sctp_chunk_stream_no(ch); stream = &q->asoc->stream; - sctp_sched_prio_sched(stream, stream->out[sid].ext); + sctp_sched_prio_sched(stream, SCTP_SO(stream, sid)->ext); } static struct sctp_chunk *sctp_sched_prio_dequeue(struct sctp_outq *q) @@ -297,7 +297,7 @@ static void sctp_sched_prio_dequeue_done(struct sctp_outq *q, * this priority. */ sid = sctp_chunk_stream_no(ch); - soute = q->asoc->stream.out[sid].ext; + soute = SCTP_SO(&q->asoc->stream, sid)->ext; prio = soute->prio_head; sctp_sched_prio_next_stream(prio); @@ -317,7 +317,7 @@ static void sctp_sched_prio_sched_all(struct sctp_stream *stream) __u16 sid; sid = sctp_chunk_stream_no(ch); - sout = &stream->out[sid]; + sout = SCTP_SO(stream, sid); if (sout->ext) sctp_sched_prio_sched(stream, sout->ext); } diff --git a/net/sctp/stream_sched_rr.c b/net/sctp/stream_sched_rr.c index 1155692448f1..52ba743fa7a7 100644 --- a/net/sctp/stream_sched_rr.c +++ b/net/sctp/stream_sched_rr.c @@ -100,7 +100,7 @@ static int sctp_sched_rr_init(struct sctp_stream *stream) static int sctp_sched_rr_init_sid(struct sctp_stream *stream, __u16 sid, gfp_t gfp) { - INIT_LIST_HEAD(&stream->out[sid].ext->rr_list); + INIT_LIST_HEAD(&SCTP_SO(stream, sid)->ext->rr_list); return 0; } @@ -120,7 +120,7 @@ static void sctp_sched_rr_enqueue(struct sctp_outq *q, ch = list_first_entry(&msg->chunks, struct sctp_chunk, frag_list); sid = sctp_chunk_stream_no(ch); stream = &q->asoc->stream; - sctp_sched_rr_sched(stream, stream->out[sid].ext); + sctp_sched_rr_sched(stream, SCTP_SO(stream, sid)->ext); } static struct sctp_chunk *sctp_sched_rr_dequeue(struct sctp_outq *q) @@ -154,7 +154,7 @@ static void sctp_sched_rr_dequeue_done(struct sctp_outq *q, /* Last chunk on that msg, move to the next stream */ sid = sctp_chunk_stream_no(ch); - soute = q->asoc->stream.out[sid].ext; + soute = SCTP_SO(&q->asoc->stream, sid)->ext; sctp_sched_rr_next_stream(&q->asoc->stream); @@ -173,7 +173,7 @@ static void sctp_sched_rr_sched_all(struct sctp_stream *stream) __u16 sid; sid = sctp_chunk_stream_no(ch); - soute = stream->out[sid].ext; + soute = SCTP_SO(stream, sid)->ext; if (soute) sctp_sched_rr_sched(stream, soute); } diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c index cfade7fdcc6d..e36f21ce7252 100644 --- a/net/smc/smc_ism.c +++ b/net/smc/smc_ism.c @@ -184,6 +184,37 @@ struct smc_ism_event_work { struct smcd_event event; }; +#define ISM_EVENT_REQUEST 0x0001 +#define ISM_EVENT_RESPONSE 0x0002 +#define ISM_EVENT_REQUEST_IR 0x00000001 +#define ISM_EVENT_CODE_TESTLINK 0x83 + +static void smcd_handle_sw_event(struct smc_ism_event_work *wrk) +{ + union { + u64 info; + struct { + u32 uid; + unsigned short vlanid; + u16 code; + }; + } ev_info; + + switch (wrk->event.code) { + case ISM_EVENT_CODE_TESTLINK: /* Activity timer */ + ev_info.info = wrk->event.info; + if (ev_info.code == ISM_EVENT_REQUEST) { + ev_info.code = ISM_EVENT_RESPONSE; + wrk->smcd->ops->signal_event(wrk->smcd, + wrk->event.tok, + ISM_EVENT_REQUEST_IR, + ISM_EVENT_CODE_TESTLINK, + ev_info.info); + } + break; + } +} + /* worker for SMC-D events */ static void smc_ism_event_work(struct work_struct *work) { @@ -196,6 +227,9 @@ static void smc_ism_event_work(struct work_struct *work) break; case ISM_EVENT_DMB: break; + case ISM_EVENT_SWR: /* Software defined event */ + smcd_handle_sw_event(wrk); + break; } kfree(wrk); } diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 83d67df33f0c..52fbe727d7c1 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -48,19 +48,13 @@ static int tls_do_decryption(struct sock *sk, struct scatterlist *sgout, char *iv_recv, size_t data_len, - struct sk_buff *skb, - gfp_t flags) + struct aead_request *aead_req) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); - struct aead_request *aead_req; - int ret; - aead_req = aead_request_alloc(ctx->aead_recv, flags); - if (!aead_req) - return -ENOMEM; - + aead_request_set_tfm(aead_req, ctx->aead_recv); aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); aead_request_set_crypt(aead_req, sgin, sgout, data_len + tls_ctx->rx.tag_size, @@ -69,8 +63,6 @@ static int tls_do_decryption(struct sock *sk, crypto_req_done, &ctx->async_wait); ret = crypto_wait_req(crypto_aead_decrypt(aead_req), &ctx->async_wait); - - aead_request_free(aead_req); return ret; } @@ -657,8 +649,132 @@ static struct sk_buff *tls_wait_data(struct sock *sk, int flags, return skb; } +/* This function decrypts the input skb into either out_iov or in out_sg + * or in skb buffers itself. The input parameter 'zc' indicates if + * zero-copy mode needs to be tried or not. With zero-copy mode, either + * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are + * NULL, then the decryption happens inside skb buffers itself, i.e. + * zero-copy gets disabled and 'zc' is updated. + */ + +static int decrypt_internal(struct sock *sk, struct sk_buff *skb, + struct iov_iter *out_iov, + struct scatterlist *out_sg, + int *chunk, bool *zc) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); + struct strp_msg *rxm = strp_msg(skb); + int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0; + struct aead_request *aead_req; + struct sk_buff *unused; + u8 *aad, *iv, *mem = NULL; + struct scatterlist *sgin = NULL; + struct scatterlist *sgout = NULL; + const int data_len = rxm->full_len - tls_ctx->rx.overhead_size; + + if (*zc && (out_iov || out_sg)) { + if (out_iov) + n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1; + else + n_sgout = sg_nents(out_sg); + } else { + n_sgout = 0; + *zc = false; + } + + n_sgin = skb_cow_data(skb, 0, &unused); + if (n_sgin < 1) + return -EBADMSG; + + /* Increment to accommodate AAD */ + n_sgin = n_sgin + 1; + + nsg = n_sgin + n_sgout; + + aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv); + mem_size = aead_size + (nsg * sizeof(struct scatterlist)); + mem_size = mem_size + TLS_AAD_SPACE_SIZE; + mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv); + + /* Allocate a single block of memory which contains + * aead_req || sgin[] || sgout[] || aad || iv. + * This order achieves correct alignment for aead_req, sgin, sgout. + */ + mem = kmalloc(mem_size, sk->sk_allocation); + if (!mem) + return -ENOMEM; + + /* Segment the allocated memory */ + aead_req = (struct aead_request *)mem; + sgin = (struct scatterlist *)(mem + aead_size); + sgout = sgin + n_sgin; + aad = (u8 *)(sgout + n_sgout); + iv = aad + TLS_AAD_SPACE_SIZE; + + /* Prepare IV */ + err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, + iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, + tls_ctx->rx.iv_size); + if (err < 0) { + kfree(mem); + return err; + } + memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); + + /* Prepare AAD */ + tls_make_aad(aad, rxm->full_len - tls_ctx->rx.overhead_size, + tls_ctx->rx.rec_seq, tls_ctx->rx.rec_seq_size, + ctx->control); + + /* Prepare sgin */ + sg_init_table(sgin, n_sgin); + sg_set_buf(&sgin[0], aad, TLS_AAD_SPACE_SIZE); + err = skb_to_sgvec(skb, &sgin[1], + rxm->offset + tls_ctx->rx.prepend_size, + rxm->full_len - tls_ctx->rx.prepend_size); + if (err < 0) { + kfree(mem); + return err; + } + + if (n_sgout) { + if (out_iov) { + sg_init_table(sgout, n_sgout); + sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE); + + *chunk = 0; + err = zerocopy_from_iter(sk, out_iov, data_len, &pages, + chunk, &sgout[1], + (n_sgout - 1), false); + if (err < 0) + goto fallback_to_reg_recv; + } else if (out_sg) { + memcpy(sgout, out_sg, n_sgout * sizeof(*sgout)); + } else { + goto fallback_to_reg_recv; + } + } else { +fallback_to_reg_recv: + sgout = sgin; + pages = 0; + *chunk = 0; + *zc = false; + } + + /* Prepare and submit AEAD request */ + err = tls_do_decryption(sk, sgin, sgout, iv, data_len, aead_req); + + /* Release the pages in case iov was mapped to pages */ + for (; pages > 0; pages--) + put_page(sg_page(&sgout[pages])); + + kfree(mem); + return err; +} + static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, - struct scatterlist *sgout, bool *zc) + struct iov_iter *dest, int *chunk, bool *zc) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); @@ -671,7 +787,7 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, return err; #endif if (!ctx->decrypted) { - err = decrypt_skb(sk, skb, sgout); + err = decrypt_internal(sk, skb, dest, NULL, chunk, zc); if (err < 0) return err; } else { @@ -690,54 +806,10 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, int decrypt_skb(struct sock *sk, struct sk_buff *skb, struct scatterlist *sgout) { - struct tls_context *tls_ctx = tls_get_ctx(sk); - struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); - char iv[TLS_CIPHER_AES_GCM_128_SALT_SIZE + MAX_IV_SIZE]; - struct scatterlist sgin_arr[MAX_SKB_FRAGS + 2]; - struct scatterlist *sgin = &sgin_arr[0]; - struct strp_msg *rxm = strp_msg(skb); - int ret, nsg = ARRAY_SIZE(sgin_arr); - struct sk_buff *unused; - - ret = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, - iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, - tls_ctx->rx.iv_size); - if (ret < 0) - return ret; - - memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); - if (!sgout) { - nsg = skb_cow_data(skb, 0, &unused) + 1; - sgin = kmalloc_array(nsg, sizeof(*sgin), sk->sk_allocation); - sgout = sgin; - } - - sg_init_table(sgin, nsg); - sg_set_buf(&sgin[0], ctx->rx_aad_ciphertext, TLS_AAD_SPACE_SIZE); - - nsg = skb_to_sgvec(skb, &sgin[1], - rxm->offset + tls_ctx->rx.prepend_size, - rxm->full_len - tls_ctx->rx.prepend_size); - if (nsg < 0) { - ret = nsg; - goto out; - } - - tls_make_aad(ctx->rx_aad_ciphertext, - rxm->full_len - tls_ctx->rx.overhead_size, - tls_ctx->rx.rec_seq, - tls_ctx->rx.rec_seq_size, - ctx->control); - - ret = tls_do_decryption(sk, sgin, sgout, iv, - rxm->full_len - tls_ctx->rx.overhead_size, - skb, sk->sk_allocation); - -out: - if (sgin != &sgin_arr[0]) - kfree(sgin); + bool zc = true; + int chunk; - return ret; + return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc); } static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb, @@ -816,43 +888,17 @@ int tls_sw_recvmsg(struct sock *sk, } if (!ctx->decrypted) { - int page_count; - int to_copy; - - page_count = iov_iter_npages(&msg->msg_iter, - MAX_SKB_FRAGS); - to_copy = rxm->full_len - tls_ctx->rx.overhead_size; - if (!is_kvec && to_copy <= len && page_count < MAX_SKB_FRAGS && - likely(!(flags & MSG_PEEK))) { - struct scatterlist sgin[MAX_SKB_FRAGS + 1]; - int pages = 0; + int to_copy = rxm->full_len - tls_ctx->rx.overhead_size; + if (!is_kvec && to_copy <= len && + likely(!(flags & MSG_PEEK))) zc = true; - sg_init_table(sgin, MAX_SKB_FRAGS + 1); - sg_set_buf(&sgin[0], ctx->rx_aad_plaintext, - TLS_AAD_SPACE_SIZE); - - err = zerocopy_from_iter(sk, &msg->msg_iter, - to_copy, &pages, - &chunk, &sgin[1], - MAX_SKB_FRAGS, false); - if (err < 0) - goto fallback_to_reg_recv; - - err = decrypt_skb_update(sk, skb, sgin, &zc); - for (; pages > 0; pages--) - put_page(sg_page(&sgin[pages])); - if (err < 0) { - tls_err_abort(sk, EBADMSG); - goto recv_end; - } - } else { -fallback_to_reg_recv: - err = decrypt_skb_update(sk, skb, NULL, &zc); - if (err < 0) { - tls_err_abort(sk, EBADMSG); - goto recv_end; - } + + err = decrypt_skb_update(sk, skb, &msg->msg_iter, + &chunk, &zc); + if (err < 0) { + tls_err_abort(sk, EBADMSG); + goto recv_end; } ctx->decrypted = true; } @@ -903,7 +949,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, int err = 0; long timeo; int chunk; - bool zc; + bool zc = false; lock_sock(sk); @@ -920,7 +966,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, } if (!ctx->decrypted) { - err = decrypt_skb_update(sk, skb, NULL, &zc); + err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc); if (err < 0) { tls_err_abort(sk, EBADMSG); diff --git a/samples/bpf/hash_func01.h b/samples/bpf/hash_func01.h new file mode 100644 index 000000000000..38255812e376 --- /dev/null +++ b/samples/bpf/hash_func01.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: LGPL-2.1 + * + * Based on Paul Hsieh's (LGPG 2.1) hash function + * From: http://www.azillionmonkeys.com/qed/hash.html + */ + +#define get16bits(d) (*((const __u16 *) (d))) + +static __always_inline +__u32 SuperFastHash (const char *data, int len, __u32 initval) { + __u32 hash = initval; + __u32 tmp; + int rem; + + if (len <= 0 || data == NULL) return 0; + + rem = len & 3; + len >>= 2; + + /* Main loop */ +#pragma clang loop unroll(full) + for (;len > 0; len--) { + hash += get16bits (data); + tmp = (get16bits (data+2) << 11) ^ hash; + hash = (hash << 16) ^ tmp; + data += 2*sizeof (__u16); + hash += hash >> 11; + } + + /* Handle end cases */ + switch (rem) { + case 3: hash += get16bits (data); + hash ^= hash << 16; + hash ^= ((signed char)data[sizeof (__u16)]) << 18; + hash += hash >> 11; + break; + case 2: hash += get16bits (data); + hash ^= hash << 11; + hash += hash >> 17; + break; + case 1: hash += (signed char)*data; + hash ^= hash << 10; + hash += hash >> 1; + } + + /* Force "avalanching" of final 127 bits */ + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + + return hash; +} diff --git a/samples/bpf/xdp_redirect_cpu_kern.c b/samples/bpf/xdp_redirect_cpu_kern.c index 8cb703671b04..a306d1c75622 100644 --- a/samples/bpf/xdp_redirect_cpu_kern.c +++ b/samples/bpf/xdp_redirect_cpu_kern.c @@ -13,8 +13,9 @@ #include <uapi/linux/bpf.h> #include "bpf_helpers.h" +#include "hash_func01.h" -#define MAX_CPUS 12 /* WARNING - sync with _user.c */ +#define MAX_CPUS 64 /* WARNING - sync with _user.c */ /* Special map type that can XDP_REDIRECT frames to another CPU */ struct bpf_map_def SEC("maps") cpu_map = { @@ -461,6 +462,108 @@ int xdp_prognum4_ddos_filter_pktgen(struct xdp_md *ctx) return bpf_redirect_map(&cpu_map, cpu_dest, 0); } +/* Hashing initval */ +#define INITVAL 15485863 + +static __always_inline +u32 get_ipv4_hash_ip_pair(struct xdp_md *ctx, u64 nh_off) +{ + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + struct iphdr *iph = data + nh_off; + u32 cpu_hash; + + if (iph + 1 > data_end) + return 0; + + cpu_hash = iph->saddr + iph->daddr; + cpu_hash = SuperFastHash((char *)&cpu_hash, 4, INITVAL + iph->protocol); + + return cpu_hash; +} + +static __always_inline +u32 get_ipv6_hash_ip_pair(struct xdp_md *ctx, u64 nh_off) +{ + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + struct ipv6hdr *ip6h = data + nh_off; + u32 cpu_hash; + + if (ip6h + 1 > data_end) + return 0; + + cpu_hash = ip6h->saddr.s6_addr32[0] + ip6h->daddr.s6_addr32[0]; + cpu_hash += ip6h->saddr.s6_addr32[1] + ip6h->daddr.s6_addr32[1]; + cpu_hash += ip6h->saddr.s6_addr32[2] + ip6h->daddr.s6_addr32[2]; + cpu_hash += ip6h->saddr.s6_addr32[3] + ip6h->daddr.s6_addr32[3]; + cpu_hash = SuperFastHash((char *)&cpu_hash, 4, INITVAL + ip6h->nexthdr); + + return cpu_hash; +} + +/* Load-Balance traffic based on hashing IP-addrs + L4-proto. The + * hashing scheme is symmetric, meaning swapping IP src/dest still hit + * same CPU. + */ +SEC("xdp_cpu_map5_lb_hash_ip_pairs") +int xdp_prognum5_lb_hash_ip_pairs(struct xdp_md *ctx) +{ + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + struct ethhdr *eth = data; + u8 ip_proto = IPPROTO_UDP; + struct datarec *rec; + u16 eth_proto = 0; + u64 l3_offset = 0; + u32 cpu_dest = 0; + u32 cpu_idx = 0; + u32 *cpu_lookup; + u32 *cpu_max; + u32 cpu_hash; + u32 key = 0; + + /* Count RX packet in map */ + rec = bpf_map_lookup_elem(&rx_cnt, &key); + if (!rec) + return XDP_ABORTED; + rec->processed++; + + cpu_max = bpf_map_lookup_elem(&cpus_count, &key); + if (!cpu_max) + return XDP_ABORTED; + + if (!(parse_eth(eth, data_end, ð_proto, &l3_offset))) + return XDP_PASS; /* Just skip */ + + /* Hash for IPv4 and IPv6 */ + switch (eth_proto) { + case ETH_P_IP: + cpu_hash = get_ipv4_hash_ip_pair(ctx, l3_offset); + break; + case ETH_P_IPV6: + cpu_hash = get_ipv6_hash_ip_pair(ctx, l3_offset); + break; + case ETH_P_ARP: /* ARP packet handled on CPU idx 0 */ + default: + cpu_hash = 0; + } + + /* Choose CPU based on hash */ + cpu_idx = cpu_hash % *cpu_max; + + cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx); + if (!cpu_lookup) + return XDP_ABORTED; + cpu_dest = *cpu_lookup; + + if (cpu_dest >= MAX_CPUS) { + rec->issue++; + return XDP_ABORTED; + } + + return bpf_redirect_map(&cpu_map, cpu_dest, 0); +} char _license[] SEC("license") = "GPL"; diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c index f6efaefd485b..9a6c7e0a6dd1 100644 --- a/samples/bpf/xdp_redirect_cpu_user.c +++ b/samples/bpf/xdp_redirect_cpu_user.c @@ -19,10 +19,10 @@ static const char *__doc__ = #include <arpa/inet.h> #include <linux/if_link.h> -#define MAX_CPUS 12 /* WARNING - sync with _kern.c */ +#define MAX_CPUS 64 /* WARNING - sync with _kern.c */ /* How many xdp_progs are defined in _kern.c */ -#define MAX_PROG 5 +#define MAX_PROG 6 /* Wanted to get rid of bpf_load.h and fake-"libbpf.h" (and instead * use bpf/libbpf.h), but cannot as (currently) needed for XDP @@ -527,7 +527,7 @@ static void stress_cpumap(void) * procedure. */ create_cpu_entry(1, 1024, 0, false); - create_cpu_entry(1, 128, 0, false); + create_cpu_entry(1, 8, 0, false); create_cpu_entry(1, 16000, 0, false); } @@ -567,7 +567,7 @@ int main(int argc, char **argv) int added_cpus = 0; int longindex = 0; int interval = 2; - int prog_num = 0; + int prog_num = 5; int add_cpu = -1; __u32 qsize; int opt; diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan index b593b36ccff8..38b2b4818e8e 100644 --- a/scripts/Makefile.ubsan +++ b/scripts/Makefile.ubsan @@ -14,10 +14,6 @@ ifdef CONFIG_UBSAN_ALIGNMENT CFLAGS_UBSAN += $(call cc-option, -fsanitize=alignment) endif -ifdef CONFIG_UBSAN_NULL - CFLAGS_UBSAN += $(call cc-option, -fsanitize=null) -endif - # -fsanitize=* options makes GCC less smart than usual and # increase number of 'maybe-uninitialized false-positives CFLAGS_UBSAN += $(call cc-option, -Wno-maybe-uninitialized) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index dd5758dc35d3..66917a4eba27 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -126,6 +126,7 @@ enum bpf_map_type { BPF_MAP_TYPE_XSKMAP, BPF_MAP_TYPE_SOCKHASH, BPF_MAP_TYPE_CGROUP_STORAGE, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, }; enum bpf_prog_type { @@ -150,6 +151,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_CGROUP_SOCK_ADDR, BPF_PROG_TYPE_LWT_SEG6LOCAL, BPF_PROG_TYPE_LIRC_MODE2, + BPF_PROG_TYPE_SK_REUSEPORT, }; enum bpf_attach_type { @@ -2091,6 +2093,24 @@ union bpf_attr { * Return * The id is returned or 0 in case the id could not be retrieved. * + * u64 bpf_skb_ancestor_cgroup_id(struct sk_buff *skb, int ancestor_level) + * Description + * Return id of cgroup v2 that is ancestor of cgroup associated + * with the *skb* at the *ancestor_level*. The root cgroup is at + * *ancestor_level* zero and each step down the hierarchy + * increments the level. If *ancestor_level* == level of cgroup + * associated with *skb*, then return value will be same as that + * of **bpf_skb_cgroup_id**\ (). + * + * The helper is useful to implement policies based on cgroups + * that are upper in hierarchy than immediate cgroup associated + * with *skb*. + * + * The format of returned id and helper limitations are same as in + * **bpf_skb_cgroup_id**\ (). + * Return + * The id is returned or 0 in case the id could not be retrieved. + * * u64 bpf_get_current_cgroup_id(void) * Return * A 64-bit integer containing the current cgroup id based @@ -2113,6 +2133,14 @@ union bpf_attr { * the shared data. * Return * Pointer to the local storage area. + * + * int bpf_sk_select_reuseport(struct sk_reuseport_md *reuse, struct bpf_map *map, void *key, u64 flags) + * Description + * Select a SO_REUSEPORT sk from a BPF_MAP_TYPE_REUSEPORT_ARRAY map + * It checks the selected sk is matching the incoming + * request in the skb. + * Return + * 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -2196,7 +2224,9 @@ union bpf_attr { FN(rc_keydown), \ FN(skb_cgroup_id), \ FN(get_current_cgroup_id), \ - FN(get_local_storage), + FN(get_local_storage), \ + FN(sk_select_reuseport), \ + FN(skb_ancestor_cgroup_id), /* integer value in 'imm' field of BPF_CALL instruction selects which helper * function eBPF program intends to call @@ -2413,6 +2443,30 @@ struct sk_msg_md { __u32 local_port; /* stored in host byte order */ }; +struct sk_reuseport_md { + /* + * Start of directly accessible data. It begins from + * the tcp/udp header. + */ + void *data; + void *data_end; /* End of directly accessible data */ + /* + * Total length of packet (starting from the tcp/udp header). + * Note that the directly accessible bytes (data_end - data) + * could be less than this "len". Those bytes could be + * indirectly read by a helper "bpf_skb_load_bytes()". + */ + __u32 len; + /* + * Eth protocol in the mac header (network byte order). e.g. + * ETH_P_IP(0x0800) and ETH_P_IPV6(0x86DD) + */ + __u32 eth_protocol; + __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ + __u32 bind_inany; /* Is sock bound to an INANY address? */ + __u32 hash; /* A hash of the packet 4 tuples */ +}; + #define BPF_TAG_SIZE 8 struct bpf_prog_info { diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 9ddc89dae962..60aa4ca8b2c5 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -92,6 +92,7 @@ int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr) attr.btf_key_type_id = create_attr->btf_key_type_id; attr.btf_value_type_id = create_attr->btf_value_type_id; attr.map_ifindex = create_attr->map_ifindex; + attr.inner_map_fd = create_attr->inner_map_fd; return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr)); } diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 0639a30a457d..6f38164b2618 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -39,6 +39,7 @@ struct bpf_create_map_attr { __u32 btf_key_type_id; __u32 btf_value_type_id; __u32 map_ifindex; + __u32 inner_map_fd; }; int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr); diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 09ecf8162f7a..cf94b0770522 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +// SPDX-License-Identifier: LGPL-2.1 /* Copyright (c) 2018 Facebook */ #include <stdlib.h> diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 43c658ccfc2b..4897e0724d4e 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: LGPL-2.1 */ /* Copyright (c) 2018 Facebook */ #ifndef __BPF_BTF_H diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 40211b51427a..2abd0f112627 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -1501,6 +1501,7 @@ static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) case BPF_PROG_TYPE_SK_MSG: case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: case BPF_PROG_TYPE_LIRC_MODE2: + case BPF_PROG_TYPE_SK_REUSEPORT: return false; case BPF_PROG_TYPE_UNSPEC: case BPF_PROG_TYPE_KPROBE: diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 17a7a5818ee1..fff7fb1285fc 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -23,7 +23,7 @@ $(TEST_CUSTOM_PROGS): $(OUTPUT)/%: %.c TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \ test_align test_verifier_log test_dev_cgroup test_tcpbpf_user \ test_sock test_btf test_sockmap test_lirc_mode2_user get_cgroup_id_user \ - test_socket_cookie test_cgroup_storage + test_socket_cookie test_cgroup_storage test_select_reuseport TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test_obj_id.o \ test_pkt_md_access.o test_xdp_redirect.o test_xdp_meta.o sockmap_parse_prog.o \ @@ -34,7 +34,8 @@ TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test test_btf_haskv.o test_btf_nokv.o test_sockmap_kern.o test_tunnel_kern.o \ test_get_stack_rawtp.o test_sockmap_kern.o test_sockhash_kern.o \ test_lwt_seg6local.o sendmsg4_prog.o sendmsg6_prog.o test_lirc_mode2_kern.o \ - get_cgroup_id_kern.o socket_cookie_prog.o + get_cgroup_id_kern.o socket_cookie_prog.o test_select_reuseport_kern.o \ + test_skb_cgroup_id_kern.o # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ @@ -45,10 +46,11 @@ TEST_PROGS := test_kmod.sh \ test_sock_addr.sh \ test_tunnel.sh \ test_lwt_seg6local.sh \ - test_lirc_mode2.sh + test_lirc_mode2.sh \ + test_skb_cgroup_id.sh # Compile but not part of 'make run_tests' -TEST_GEN_PROGS_EXTENDED = test_libbpf_open test_sock_addr +TEST_GEN_PROGS_EXTENDED = test_libbpf_open test_sock_addr test_skb_cgroup_id_user include ../lib.mk @@ -59,6 +61,7 @@ $(TEST_GEN_PROGS): $(BPFOBJ) $(TEST_GEN_PROGS_EXTENDED): $(OUTPUT)/libbpf.a $(OUTPUT)/test_dev_cgroup: cgroup_helpers.c +$(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c $(OUTPUT)/test_sock: cgroup_helpers.c $(OUTPUT)/test_sock_addr: cgroup_helpers.c $(OUTPUT)/test_socket_cookie: cgroup_helpers.c diff --git a/tools/testing/selftests/bpf/bpf_helpers.h b/tools/testing/selftests/bpf/bpf_helpers.h index 9ba1c72d7cf5..e4be7730222d 100644 --- a/tools/testing/selftests/bpf/bpf_helpers.h +++ b/tools/testing/selftests/bpf/bpf_helpers.h @@ -111,6 +111,8 @@ static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = static int (*bpf_skb_get_xfrm_state)(void *ctx, int index, void *state, int size, int flags) = (void *) BPF_FUNC_skb_get_xfrm_state; +static int (*bpf_sk_select_reuseport)(void *ctx, void *map, void *key, __u32 flags) = + (void *) BPF_FUNC_sk_select_reuseport; static int (*bpf_get_stack)(void *ctx, void *buf, int size, int flags) = (void *) BPF_FUNC_get_stack; static int (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, @@ -137,6 +139,10 @@ static unsigned long long (*bpf_get_current_cgroup_id)(void) = (void *) BPF_FUNC_get_current_cgroup_id; static void *(*bpf_get_local_storage)(void *map, unsigned long long flags) = (void *) BPF_FUNC_get_local_storage; +static unsigned long long (*bpf_skb_cgroup_id)(void *ctx) = + (void *) BPF_FUNC_skb_cgroup_id; +static unsigned long long (*bpf_skb_ancestor_cgroup_id)(void *ctx, int level) = + (void *) BPF_FUNC_skb_ancestor_cgroup_id; /* llvm builtin functions that eBPF C program may use to * emit BPF_LD_ABS and BPF_LD_IND instructions @@ -173,6 +179,8 @@ struct bpf_map_def { static int (*bpf_skb_load_bytes)(void *ctx, int off, void *to, int len) = (void *) BPF_FUNC_skb_load_bytes; +static int (*bpf_skb_load_bytes_relative)(void *ctx, int off, void *to, int len, __u32 start_header) = + (void *) BPF_FUNC_skb_load_bytes_relative; static int (*bpf_skb_store_bytes)(void *ctx, int off, void *from, int len, int flags) = (void *) BPF_FUNC_skb_store_bytes; static int (*bpf_l3_csum_replace)(void *ctx, int off, int from, int to, int flags) = diff --git a/tools/testing/selftests/bpf/bpf_util.h b/tools/testing/selftests/bpf/bpf_util.h index d0811b3d6a6f..315a44fa32af 100644 --- a/tools/testing/selftests/bpf/bpf_util.h +++ b/tools/testing/selftests/bpf/bpf_util.h @@ -44,4 +44,8 @@ static inline unsigned int bpf_num_possible_cpus(void) name[bpf_num_possible_cpus()] #define bpf_percpu(name, cpu) name[(cpu)].v +#ifndef ARRAY_SIZE +# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + #endif /* __BPF_UTIL__ */ diff --git a/tools/testing/selftests/bpf/test_align.c b/tools/testing/selftests/bpf/test_align.c index 6b1b302310fe..5f377ec53f2f 100644 --- a/tools/testing/selftests/bpf/test_align.c +++ b/tools/testing/selftests/bpf/test_align.c @@ -18,10 +18,7 @@ #include "../../../include/linux/filter.h" #include "bpf_rlimit.h" - -#ifndef ARRAY_SIZE -# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) -#endif +#include "bpf_util.h" #define MAX_INSNS 512 #define MAX_MATCHES 16 diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c index ffdd27737c9e..6b5cfeb7a9cc 100644 --- a/tools/testing/selftests/bpf/test_btf.c +++ b/tools/testing/selftests/bpf/test_btf.c @@ -19,6 +19,7 @@ #include <bpf/btf.h> #include "bpf_rlimit.h" +#include "bpf_util.h" static uint32_t pass_cnt; static uint32_t error_cnt; @@ -93,10 +94,6 @@ static int __base_pr(const char *format, ...) #define MAX_NR_RAW_TYPES 1024 #define BTF_LOG_BUF_SIZE 65535 -#ifndef ARRAY_SIZE -# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) -#endif - static struct args { unsigned int raw_test_num; unsigned int file_test_num; @@ -131,6 +128,8 @@ struct btf_raw_test { __u32 max_entries; bool btf_load_err; bool map_create_err; + bool ordered_map; + bool lossless_map; int hdr_len_delta; int type_off_delta; int str_off_delta; @@ -2093,8 +2092,7 @@ struct pprint_mapv { } aenum; }; -static struct btf_raw_test pprint_test = { - .descr = "BTF pretty print test #1", +static struct btf_raw_test pprint_test_template = { .raw_types = { /* unsighed char */ /* [1] */ BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 8, 1), @@ -2146,8 +2144,6 @@ static struct btf_raw_test pprint_test = { }, .str_sec = "\0unsigned char\0unsigned short\0unsigned int\0int\0unsigned long long\0uint8_t\0uint16_t\0uint32_t\0int32_t\0uint64_t\0ui64\0ui8a\0ENUM_ZERO\0ENUM_ONE\0ENUM_TWO\0ENUM_THREE\0pprint_mapv\0ui32\0ui16\0si32\0unused_bits2a\0bits28\0unused_bits2b\0aenum", .str_sec_size = sizeof("\0unsigned char\0unsigned short\0unsigned int\0int\0unsigned long long\0uint8_t\0uint16_t\0uint32_t\0int32_t\0uint64_t\0ui64\0ui8a\0ENUM_ZERO\0ENUM_ONE\0ENUM_TWO\0ENUM_THREE\0pprint_mapv\0ui32\0ui16\0si32\0unused_bits2a\0bits28\0unused_bits2b\0aenum"), - .map_type = BPF_MAP_TYPE_ARRAY, - .map_name = "pprint_test", .key_size = sizeof(unsigned int), .value_size = sizeof(struct pprint_mapv), .key_type_id = 3, /* unsigned int */ @@ -2155,6 +2151,40 @@ static struct btf_raw_test pprint_test = { .max_entries = 128 * 1024, }; +static struct btf_pprint_test_meta { + const char *descr; + enum bpf_map_type map_type; + const char *map_name; + bool ordered_map; + bool lossless_map; +} pprint_tests_meta[] = { +{ + .descr = "BTF pretty print array", + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "pprint_test_array", + .ordered_map = true, + .lossless_map = true, +}, + +{ + .descr = "BTF pretty print hash", + .map_type = BPF_MAP_TYPE_HASH, + .map_name = "pprint_test_hash", + .ordered_map = false, + .lossless_map = true, +}, + +{ + .descr = "BTF pretty print lru hash", + .map_type = BPF_MAP_TYPE_LRU_HASH, + .map_name = "pprint_test_lru_hash", + .ordered_map = false, + .lossless_map = false, +}, + +}; + + static void set_pprint_mapv(struct pprint_mapv *v, uint32_t i) { v->ui32 = i; @@ -2166,10 +2196,12 @@ static void set_pprint_mapv(struct pprint_mapv *v, uint32_t i) v->aenum = i & 0x03; } -static int test_pprint(void) +static int do_test_pprint(void) { - const struct btf_raw_test *test = &pprint_test; + const struct btf_raw_test *test = &pprint_test_template; struct bpf_create_map_attr create_attr = {}; + unsigned int key, nr_read_elems; + bool ordered_map, lossless_map; int map_fd = -1, btf_fd = -1; struct pprint_mapv mapv = {}; unsigned int raw_btf_size; @@ -2178,7 +2210,6 @@ static int test_pprint(void) char pin_path[255]; size_t line_len = 0; char *line = NULL; - unsigned int key; uint8_t *raw_btf; ssize_t nread; int err, ret; @@ -2251,14 +2282,18 @@ static int test_pprint(void) goto done; } - key = 0; + nr_read_elems = 0; + ordered_map = test->ordered_map; + lossless_map = test->lossless_map; do { ssize_t nexpected_line; + unsigned int next_key; - set_pprint_mapv(&mapv, key); + next_key = ordered_map ? nr_read_elems : atoi(line); + set_pprint_mapv(&mapv, next_key); nexpected_line = snprintf(expected_line, sizeof(expected_line), "%u: {%u,0,%d,0x%x,0x%x,0x%x,{%lu|[%u,%u,%u,%u,%u,%u,%u,%u]},%s}\n", - key, + next_key, mapv.ui32, mapv.si32, mapv.unused_bits2a, mapv.bits28, mapv.unused_bits2b, mapv.ui64, @@ -2281,11 +2316,12 @@ static int test_pprint(void) } nread = getline(&line, &line_len, pin_file); - } while (++key < test->max_entries && nread > 0); + } while (++nr_read_elems < test->max_entries && nread > 0); - if (CHECK(key < test->max_entries, - "Unexpected EOF. key:%u test->max_entries:%u", - key, test->max_entries)) { + if (lossless_map && + CHECK(nr_read_elems < test->max_entries, + "Unexpected EOF. nr_read_elems:%u test->max_entries:%u", + nr_read_elems, test->max_entries)) { err = -1; goto done; } @@ -2314,6 +2350,24 @@ done: return err; } +static int test_pprint(void) +{ + unsigned int i; + int err = 0; + + for (i = 0; i < ARRAY_SIZE(pprint_tests_meta); i++) { + pprint_test_template.descr = pprint_tests_meta[i].descr; + pprint_test_template.map_type = pprint_tests_meta[i].map_type; + pprint_test_template.map_name = pprint_tests_meta[i].map_name; + pprint_test_template.ordered_map = pprint_tests_meta[i].ordered_map; + pprint_test_template.lossless_map = pprint_tests_meta[i].lossless_map; + + err |= count_result(do_test_pprint()); + } + + return err; +} + static void usage(const char *cmd) { fprintf(stderr, "Usage: %s [-l] [[-r test_num (1 - %zu)] | [-g test_num (1 - %zu)] | [-f test_num (1 - %zu)] | [-p]]\n", @@ -2409,7 +2463,7 @@ int main(int argc, char **argv) err |= test_file(); if (args.pprint_test) - err |= count_result(test_pprint()); + err |= test_pprint(); if (args.raw_test || args.get_info_test || args.file_test || args.pprint_test) diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index 6c253343a6f9..4b7c74f5faa7 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -17,7 +17,8 @@ #include <stdlib.h> #include <sys/wait.h> - +#include <sys/socket.h> +#include <netinet/in.h> #include <linux/bpf.h> #include <bpf/bpf.h> @@ -26,8 +27,21 @@ #include "bpf_util.h" #include "bpf_rlimit.h" +#ifndef ENOTSUPP +#define ENOTSUPP 524 +#endif + static int map_flags; +#define CHECK(condition, tag, format...) ({ \ + int __ret = !!(condition); \ + if (__ret) { \ + printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag); \ + printf(format); \ + exit(-1); \ + } \ +}) + static void test_hashmap(int task, void *data) { long long key, next_key, first_key, value; @@ -1150,6 +1164,250 @@ static void test_map_wronly(void) assert(bpf_map_get_next_key(fd, &key, &value) == -1 && errno == EPERM); } +static void prepare_reuseport_grp(int type, int map_fd, + __s64 *fds64, __u64 *sk_cookies, + unsigned int n) +{ + socklen_t optlen, addrlen; + struct sockaddr_in6 s6; + const __u32 index0 = 0; + const int optval = 1; + unsigned int i; + u64 sk_cookie; + __s64 fd64; + int err; + + s6.sin6_family = AF_INET6; + s6.sin6_addr = in6addr_any; + s6.sin6_port = 0; + addrlen = sizeof(s6); + optlen = sizeof(sk_cookie); + + for (i = 0; i < n; i++) { + fd64 = socket(AF_INET6, type, 0); + CHECK(fd64 == -1, "socket()", + "sock_type:%d fd64:%lld errno:%d\n", + type, fd64, errno); + + err = setsockopt(fd64, SOL_SOCKET, SO_REUSEPORT, + &optval, sizeof(optval)); + CHECK(err == -1, "setsockopt(SO_REUSEEPORT)", + "err:%d errno:%d\n", err, errno); + + /* reuseport_array does not allow unbound sk */ + err = bpf_map_update_elem(map_fd, &index0, &fd64, + BPF_ANY); + CHECK(err != -1 || errno != EINVAL, + "reuseport array update unbound sk", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + + err = bind(fd64, (struct sockaddr *)&s6, sizeof(s6)); + CHECK(err == -1, "bind()", + "sock_type:%d err:%d errno:%d\n", type, err, errno); + + if (i == 0) { + err = getsockname(fd64, (struct sockaddr *)&s6, + &addrlen); + CHECK(err == -1, "getsockname()", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + } + + err = getsockopt(fd64, SOL_SOCKET, SO_COOKIE, &sk_cookie, + &optlen); + CHECK(err == -1, "getsockopt(SO_COOKIE)", + "sock_type:%d err:%d errno:%d\n", type, err, errno); + + if (type == SOCK_STREAM) { + /* + * reuseport_array does not allow + * non-listening tcp sk. + */ + err = bpf_map_update_elem(map_fd, &index0, &fd64, + BPF_ANY); + CHECK(err != -1 || errno != EINVAL, + "reuseport array update non-listening sk", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + err = listen(fd64, 0); + CHECK(err == -1, "listen()", + "sock_type:%d, err:%d errno:%d\n", + type, err, errno); + } + + fds64[i] = fd64; + sk_cookies[i] = sk_cookie; + } +} + +static void test_reuseport_array(void) +{ +#define REUSEPORT_FD_IDX(err, last) ({ (err) ? last : !last; }) + + const __u32 array_size = 4, index0 = 0, index3 = 3; + int types[2] = { SOCK_STREAM, SOCK_DGRAM }, type; + __u64 grpa_cookies[2], sk_cookie, map_cookie; + __s64 grpa_fds64[2] = { -1, -1 }, fd64 = -1; + const __u32 bad_index = array_size; + int map_fd, err, t, f; + __u32 fds_idx = 0; + int fd; + + map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, + sizeof(__u32), sizeof(__u64), array_size, 0); + CHECK(map_fd == -1, "reuseport array create", + "map_fd:%d, errno:%d\n", map_fd, errno); + + /* Test lookup/update/delete with invalid index */ + err = bpf_map_delete_elem(map_fd, &bad_index); + CHECK(err != -1 || errno != E2BIG, "reuseport array del >=max_entries", + "err:%d errno:%d\n", err, errno); + + err = bpf_map_update_elem(map_fd, &bad_index, &fd64, BPF_ANY); + CHECK(err != -1 || errno != E2BIG, + "reuseport array update >=max_entries", + "err:%d errno:%d\n", err, errno); + + err = bpf_map_lookup_elem(map_fd, &bad_index, &map_cookie); + CHECK(err != -1 || errno != ENOENT, + "reuseport array update >=max_entries", + "err:%d errno:%d\n", err, errno); + + /* Test lookup/delete non existence elem */ + err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); + CHECK(err != -1 || errno != ENOENT, + "reuseport array lookup not-exist elem", + "err:%d errno:%d\n", err, errno); + err = bpf_map_delete_elem(map_fd, &index3); + CHECK(err != -1 || errno != ENOENT, + "reuseport array del not-exist elem", + "err:%d errno:%d\n", err, errno); + + for (t = 0; t < ARRAY_SIZE(types); t++) { + type = types[t]; + + prepare_reuseport_grp(type, map_fd, grpa_fds64, + grpa_cookies, ARRAY_SIZE(grpa_fds64)); + + /* Test BPF_* update flags */ + /* BPF_EXIST failure case */ + err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], + BPF_EXIST); + CHECK(err != -1 || errno != ENOENT, + "reuseport array update empty elem BPF_EXIST", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + fds_idx = REUSEPORT_FD_IDX(err, fds_idx); + + /* BPF_NOEXIST success case */ + err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], + BPF_NOEXIST); + CHECK(err == -1, + "reuseport array update empty elem BPF_NOEXIST", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + fds_idx = REUSEPORT_FD_IDX(err, fds_idx); + + /* BPF_EXIST success case. */ + err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], + BPF_EXIST); + CHECK(err == -1, + "reuseport array update same elem BPF_EXIST", + "sock_type:%d err:%d errno:%d\n", type, err, errno); + fds_idx = REUSEPORT_FD_IDX(err, fds_idx); + + /* BPF_NOEXIST failure case */ + err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], + BPF_NOEXIST); + CHECK(err != -1 || errno != EEXIST, + "reuseport array update non-empty elem BPF_NOEXIST", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + fds_idx = REUSEPORT_FD_IDX(err, fds_idx); + + /* BPF_ANY case (always succeed) */ + err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], + BPF_ANY); + CHECK(err == -1, + "reuseport array update same sk with BPF_ANY", + "sock_type:%d err:%d errno:%d\n", type, err, errno); + + fd64 = grpa_fds64[fds_idx]; + sk_cookie = grpa_cookies[fds_idx]; + + /* The same sk cannot be added to reuseport_array twice */ + err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_ANY); + CHECK(err != -1 || errno != EBUSY, + "reuseport array update same sk with same index", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + + err = bpf_map_update_elem(map_fd, &index0, &fd64, BPF_ANY); + CHECK(err != -1 || errno != EBUSY, + "reuseport array update same sk with different index", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + + /* Test delete elem */ + err = bpf_map_delete_elem(map_fd, &index3); + CHECK(err == -1, "reuseport array delete sk", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + + /* Add it back with BPF_NOEXIST */ + err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); + CHECK(err == -1, + "reuseport array re-add with BPF_NOEXIST after del", + "sock_type:%d err:%d errno:%d\n", type, err, errno); + + /* Test cookie */ + err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); + CHECK(err == -1 || sk_cookie != map_cookie, + "reuseport array lookup re-added sk", + "sock_type:%d err:%d errno:%d sk_cookie:0x%llx map_cookie:0x%llxn", + type, err, errno, sk_cookie, map_cookie); + + /* Test elem removed by close() */ + for (f = 0; f < ARRAY_SIZE(grpa_fds64); f++) + close(grpa_fds64[f]); + err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); + CHECK(err != -1 || errno != ENOENT, + "reuseport array lookup after close()", + "sock_type:%d err:%d errno:%d\n", + type, err, errno); + } + + /* Test SOCK_RAW */ + fd64 = socket(AF_INET6, SOCK_RAW, IPPROTO_UDP); + CHECK(fd64 == -1, "socket(SOCK_RAW)", "err:%d errno:%d\n", + err, errno); + err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); + CHECK(err != -1 || errno != ENOTSUPP, "reuseport array update SOCK_RAW", + "err:%d errno:%d\n", err, errno); + close(fd64); + + /* Close the 64 bit value map */ + close(map_fd); + + /* Test 32 bit fd */ + map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, + sizeof(__u32), sizeof(__u32), array_size, 0); + CHECK(map_fd == -1, "reuseport array create", + "map_fd:%d, errno:%d\n", map_fd, errno); + prepare_reuseport_grp(SOCK_STREAM, map_fd, &fd64, &sk_cookie, 1); + fd = fd64; + err = bpf_map_update_elem(map_fd, &index3, &fd, BPF_NOEXIST); + CHECK(err == -1, "reuseport array update 32 bit fd", + "err:%d errno:%d\n", err, errno); + err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); + CHECK(err != -1 || errno != ENOSPC, + "reuseport array lookup 32 bit fd", + "err:%d errno:%d\n", err, errno); + close(fd); + close(map_fd); +} + static void run_all_tests(void) { test_hashmap(0, NULL); @@ -1170,6 +1428,8 @@ static void run_all_tests(void) test_map_rdonly(); test_map_wronly(); + + test_reuseport_array(); } int main(void) diff --git a/tools/testing/selftests/bpf/test_select_reuseport.c b/tools/testing/selftests/bpf/test_select_reuseport.c new file mode 100644 index 000000000000..75646d9b34aa --- /dev/null +++ b/tools/testing/selftests/bpf/test_select_reuseport.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2018 Facebook */ + +#include <stdlib.h> +#include <unistd.h> +#include <stdbool.h> +#include <string.h> +#include <errno.h> +#include <assert.h> +#include <fcntl.h> +#include <linux/bpf.h> +#include <linux/err.h> +#include <linux/types.h> +#include <linux/if_ether.h> +#include <sys/types.h> +#include <sys/epoll.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <bpf/bpf.h> +#include <bpf/libbpf.h> +#include "bpf_rlimit.h" +#include "bpf_util.h" +#include "test_select_reuseport_common.h" + +#define MIN_TCPHDR_LEN 20 +#define UDPHDR_LEN 8 + +#define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" +#define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" +#define REUSEPORT_ARRAY_SIZE 32 + +static int result_map, tmp_index_ovr_map, linum_map, data_check_map; +static enum result expected_results[NR_RESULTS]; +static int sk_fds[REUSEPORT_ARRAY_SIZE]; +static int reuseport_array, outer_map; +static int select_by_skb_data_prog; +static int saved_tcp_syncookie; +static struct bpf_object *obj; +static int saved_tcp_fo; +static __u32 index_zero; +static int epfd; + +static union sa46 { + struct sockaddr_in6 v6; + struct sockaddr_in v4; + sa_family_t family; +} srv_sa; + +#define CHECK(condition, tag, format...) ({ \ + int __ret = !!(condition); \ + if (__ret) { \ + printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag); \ + printf(format); \ + exit(-1); \ + } \ +}) + +static void create_maps(void) +{ + struct bpf_create_map_attr attr = {}; + + /* Creating reuseport_array */ + attr.name = "reuseport_array"; + attr.map_type = BPF_MAP_TYPE_REUSEPORT_SOCKARRAY; + attr.key_size = sizeof(__u32); + attr.value_size = sizeof(__u32); + attr.max_entries = REUSEPORT_ARRAY_SIZE; + + reuseport_array = bpf_create_map_xattr(&attr); + CHECK(reuseport_array == -1, "creating reuseport_array", + "reuseport_array:%d errno:%d\n", reuseport_array, errno); + + /* Creating outer_map */ + attr.name = "outer_map"; + attr.map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS; + attr.key_size = sizeof(__u32); + attr.value_size = sizeof(__u32); + attr.max_entries = 1; + attr.inner_map_fd = reuseport_array; + outer_map = bpf_create_map_xattr(&attr); + CHECK(outer_map == -1, "creating outer_map", + "outer_map:%d errno:%d\n", outer_map, errno); +} + +static void prepare_bpf_obj(void) +{ + struct bpf_program *prog; + struct bpf_map *map; + int err; + struct bpf_object_open_attr attr = { + .file = "test_select_reuseport_kern.o", + .prog_type = BPF_PROG_TYPE_SK_REUSEPORT, + }; + + obj = bpf_object__open_xattr(&attr); + CHECK(IS_ERR_OR_NULL(obj), "open test_select_reuseport_kern.o", + "obj:%p PTR_ERR(obj):%ld\n", obj, PTR_ERR(obj)); + + prog = bpf_program__next(NULL, obj); + CHECK(!prog, "get first bpf_program", "!prog\n"); + bpf_program__set_type(prog, attr.prog_type); + + map = bpf_object__find_map_by_name(obj, "outer_map"); + CHECK(!map, "find outer_map", "!map\n"); + err = bpf_map__reuse_fd(map, outer_map); + CHECK(err, "reuse outer_map", "err:%d\n", err); + + err = bpf_object__load(obj); + CHECK(err, "load bpf_object", "err:%d\n", err); + + select_by_skb_data_prog = bpf_program__fd(prog); + CHECK(select_by_skb_data_prog == -1, "get prog fd", + "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); + + map = bpf_object__find_map_by_name(obj, "result_map"); + CHECK(!map, "find result_map", "!map\n"); + result_map = bpf_map__fd(map); + CHECK(result_map == -1, "get result_map fd", + "result_map:%d\n", result_map); + + map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); + CHECK(!map, "find tmp_index_ovr_map", "!map\n"); + tmp_index_ovr_map = bpf_map__fd(map); + CHECK(tmp_index_ovr_map == -1, "get tmp_index_ovr_map fd", + "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); + + map = bpf_object__find_map_by_name(obj, "linum_map"); + CHECK(!map, "find linum_map", "!map\n"); + linum_map = bpf_map__fd(map); + CHECK(linum_map == -1, "get linum_map fd", + "linum_map:%d\n", linum_map); + + map = bpf_object__find_map_by_name(obj, "data_check_map"); + CHECK(!map, "find data_check_map", "!map\n"); + data_check_map = bpf_map__fd(map); + CHECK(data_check_map == -1, "get data_check_map fd", + "data_check_map:%d\n", data_check_map); +} + +static void sa46_init_loopback(union sa46 *sa, sa_family_t family) +{ + memset(sa, 0, sizeof(*sa)); + sa->family = family; + if (sa->family == AF_INET6) + sa->v6.sin6_addr = in6addr_loopback; + else + sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); +} + +static void sa46_init_inany(union sa46 *sa, sa_family_t family) +{ + memset(sa, 0, sizeof(*sa)); + sa->family = family; + if (sa->family == AF_INET6) + sa->v6.sin6_addr = in6addr_any; + else + sa->v4.sin_addr.s_addr = INADDR_ANY; +} + +static int read_int_sysctl(const char *sysctl) +{ + char buf[16]; + int fd, ret; + + fd = open(sysctl, 0); + CHECK(fd == -1, "open(sysctl)", "sysctl:%s fd:%d errno:%d\n", + sysctl, fd, errno); + + ret = read(fd, buf, sizeof(buf)); + CHECK(ret <= 0, "read(sysctl)", "sysctl:%s ret:%d errno:%d\n", + sysctl, ret, errno); + close(fd); + + return atoi(buf); +} + +static void write_int_sysctl(const char *sysctl, int v) +{ + int fd, ret, size; + char buf[16]; + + fd = open(sysctl, O_RDWR); + CHECK(fd == -1, "open(sysctl)", "sysctl:%s fd:%d errno:%d\n", + sysctl, fd, errno); + + size = snprintf(buf, sizeof(buf), "%d", v); + ret = write(fd, buf, size); + CHECK(ret != size, "write(sysctl)", + "sysctl:%s ret:%d size:%d errno:%d\n", sysctl, ret, size, errno); + close(fd); +} + +static void restore_sysctls(void) +{ + write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo); + write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie); +} + +static void enable_fastopen(void) +{ + int fo; + + fo = read_int_sysctl(TCP_FO_SYSCTL); + write_int_sysctl(TCP_FO_SYSCTL, fo | 7); +} + +static void enable_syncookie(void) +{ + write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); +} + +static void disable_syncookie(void) +{ + write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); +} + +static __u32 get_linum(void) +{ + __u32 linum; + int err; + + err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); + CHECK(err == -1, "lookup_elem(linum_map)", "err:%d errno:%d\n", + err, errno); + + return linum; +} + +static void check_data(int type, sa_family_t family, const struct cmd *cmd, + int cli_fd) +{ + struct data_check expected = {}, result; + union sa46 cli_sa; + socklen_t addrlen; + int err; + + addrlen = sizeof(cli_sa); + err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, + &addrlen); + CHECK(err == -1, "getsockname(cli_fd)", "err:%d errno:%d\n", + err, errno); + + err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); + CHECK(err == -1, "lookup_elem(data_check_map)", "err:%d errno:%d\n", + err, errno); + + if (type == SOCK_STREAM) { + expected.len = MIN_TCPHDR_LEN; + expected.ip_protocol = IPPROTO_TCP; + } else { + expected.len = UDPHDR_LEN; + expected.ip_protocol = IPPROTO_UDP; + } + + if (family == AF_INET6) { + expected.eth_protocol = htons(ETH_P_IPV6); + expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] && + !srv_sa.v6.sin6_addr.s6_addr32[2] && + !srv_sa.v6.sin6_addr.s6_addr32[1] && + !srv_sa.v6.sin6_addr.s6_addr32[0]; + + memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32, + sizeof(cli_sa.v6.sin6_addr)); + memcpy(&expected.skb_addrs[4], &in6addr_loopback, + sizeof(in6addr_loopback)); + expected.skb_ports[0] = cli_sa.v6.sin6_port; + expected.skb_ports[1] = srv_sa.v6.sin6_port; + } else { + expected.eth_protocol = htons(ETH_P_IP); + expected.bind_inany = !srv_sa.v4.sin_addr.s_addr; + + expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr; + expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); + expected.skb_ports[0] = cli_sa.v4.sin_port; + expected.skb_ports[1] = srv_sa.v4.sin_port; + } + + if (memcmp(&result, &expected, offsetof(struct data_check, + equal_check_end))) { + printf("unexpected data_check\n"); + printf(" result: (0x%x, %u, %u)\n", + result.eth_protocol, result.ip_protocol, + result.bind_inany); + printf("expected: (0x%x, %u, %u)\n", + expected.eth_protocol, expected.ip_protocol, + expected.bind_inany); + CHECK(1, "data_check result != expected", + "bpf_prog_linum:%u\n", get_linum()); + } + + CHECK(!result.hash, "data_check result.hash empty", + "result.hash:%u", result.hash); + + expected.len += cmd ? sizeof(*cmd) : 0; + if (type == SOCK_STREAM) + CHECK(expected.len > result.len, "expected.len > result.len", + "expected.len:%u result.len:%u bpf_prog_linum:%u\n", + expected.len, result.len, get_linum()); + else + CHECK(expected.len != result.len, "expected.len != result.len", + "expected.len:%u result.len:%u bpf_prog_linum:%u\n", + expected.len, result.len, get_linum()); +} + +static void check_results(void) +{ + __u32 results[NR_RESULTS]; + __u32 i, broken = 0; + int err; + + for (i = 0; i < NR_RESULTS; i++) { + err = bpf_map_lookup_elem(result_map, &i, &results[i]); + CHECK(err == -1, "lookup_elem(result_map)", + "i:%u err:%d errno:%d\n", i, err, errno); + } + + for (i = 0; i < NR_RESULTS; i++) { + if (results[i] != expected_results[i]) { + broken = i; + break; + } + } + + if (i == NR_RESULTS) + return; + + printf("unexpected result\n"); + printf(" result: ["); + printf("%u", results[0]); + for (i = 1; i < NR_RESULTS; i++) + printf(", %u", results[i]); + printf("]\n"); + + printf("expected: ["); + printf("%u", expected_results[0]); + for (i = 1; i < NR_RESULTS; i++) + printf(", %u", expected_results[i]); + printf("]\n"); + + CHECK(expected_results[broken] != results[broken], + "unexpected result", + "expected_results[%u] != results[%u] bpf_prog_linum:%u\n", + broken, broken, get_linum()); +} + +static int send_data(int type, sa_family_t family, void *data, size_t len, + enum result expected) +{ + union sa46 cli_sa; + int fd, err; + + fd = socket(family, type, 0); + CHECK(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); + + sa46_init_loopback(&cli_sa, family); + err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); + CHECK(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); + + err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, + sizeof(srv_sa)); + CHECK(err != len && expected >= PASS, + "sendto()", "family:%u err:%d errno:%d expected:%d\n", + family, err, errno, expected); + + return fd; +} + +static void do_test(int type, sa_family_t family, struct cmd *cmd, + enum result expected) +{ + int nev, srv_fd, cli_fd; + struct epoll_event ev; + struct cmd rcv_cmd; + ssize_t nread; + + cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, + expected); + nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); + CHECK((nev <= 0 && expected >= PASS) || + (nev > 0 && expected < PASS), + "nev <> expected", + "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", + nev, expected, type, family, + cmd ? cmd->reuseport_index : -1, + cmd ? cmd->pass_on_failure : -1); + check_results(); + check_data(type, family, cmd, cli_fd); + + if (expected < PASS) + return; + + CHECK(expected != PASS_ERR_SK_SELECT_REUSEPORT && + cmd->reuseport_index != ev.data.u32, + "check cmd->reuseport_index", + "cmd:(%u, %u) ev.data.u32:%u\n", + cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); + + srv_fd = sk_fds[ev.data.u32]; + if (type == SOCK_STREAM) { + int new_fd = accept(srv_fd, NULL, 0); + + CHECK(new_fd == -1, "accept(srv_fd)", + "ev.data.u32:%u new_fd:%d errno:%d\n", + ev.data.u32, new_fd, errno); + + nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); + CHECK(nread != sizeof(rcv_cmd), + "recv(new_fd)", + "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", + ev.data.u32, nread, sizeof(rcv_cmd), errno); + + close(new_fd); + } else { + nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); + CHECK(nread != sizeof(rcv_cmd), + "recv(sk_fds)", + "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", + ev.data.u32, nread, sizeof(rcv_cmd), errno); + } + + close(cli_fd); +} + +static void test_err_inner_map(int type, sa_family_t family) +{ + struct cmd cmd = { + .reuseport_index = 0, + .pass_on_failure = 0, + }; + + printf("%s: ", __func__); + expected_results[DROP_ERR_INNER_MAP]++; + do_test(type, family, &cmd, DROP_ERR_INNER_MAP); + printf("OK\n"); +} + +static void test_err_skb_data(int type, sa_family_t family) +{ + printf("%s: ", __func__); + expected_results[DROP_ERR_SKB_DATA]++; + do_test(type, family, NULL, DROP_ERR_SKB_DATA); + printf("OK\n"); +} + +static void test_err_sk_select_port(int type, sa_family_t family) +{ + struct cmd cmd = { + .reuseport_index = REUSEPORT_ARRAY_SIZE, + .pass_on_failure = 0, + }; + + printf("%s: ", __func__); + expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; + do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); + printf("OK\n"); +} + +static void test_pass(int type, sa_family_t family) +{ + struct cmd cmd; + int i; + + printf("%s: ", __func__); + cmd.pass_on_failure = 0; + for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { + expected_results[PASS]++; + cmd.reuseport_index = i; + do_test(type, family, &cmd, PASS); + } + printf("OK\n"); +} + +static void test_syncookie(int type, sa_family_t family) +{ + int err, tmp_index = 1; + struct cmd cmd = { + .reuseport_index = 0, + .pass_on_failure = 0, + }; + + if (type != SOCK_STREAM) + return; + + printf("%s: ", __func__); + /* + * +1 for TCP-SYN and + * +1 for the TCP-ACK (ack the syncookie) + */ + expected_results[PASS] += 2; + enable_syncookie(); + /* + * Simulate TCP-SYN and TCP-ACK are handled by two different sk: + * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the + * tmp_index_ovr_map + * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index + * is from the cmd.reuseport_index + */ + err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, + &tmp_index, BPF_ANY); + CHECK(err == -1, "update_elem(tmp_index_ovr_map, 0, 1)", + "err:%d errno:%d\n", err, errno); + do_test(type, family, &cmd, PASS); + err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, + &tmp_index); + CHECK(err == -1 || tmp_index != -1, + "lookup_elem(tmp_index_ovr_map)", + "err:%d errno:%d tmp_index:%d\n", + err, errno, tmp_index); + disable_syncookie(); + printf("OK\n"); +} + +static void test_pass_on_err(int type, sa_family_t family) +{ + struct cmd cmd = { + .reuseport_index = REUSEPORT_ARRAY_SIZE, + .pass_on_failure = 1, + }; + + printf("%s: ", __func__); + expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; + do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); + printf("OK\n"); +} + +static void prepare_sk_fds(int type, sa_family_t family, bool inany) +{ + const int first = REUSEPORT_ARRAY_SIZE - 1; + int i, err, optval = 1; + struct epoll_event ev; + socklen_t addrlen; + + if (inany) + sa46_init_inany(&srv_sa, family); + else + sa46_init_loopback(&srv_sa, family); + addrlen = sizeof(srv_sa); + + /* + * The sk_fds[] is filled from the back such that the order + * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. + */ + for (i = first; i >= 0; i--) { + sk_fds[i] = socket(family, type, 0); + CHECK(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", + i, sk_fds[i], errno); + err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, + &optval, sizeof(optval)); + CHECK(err == -1, "setsockopt(SO_REUSEPORT)", + "sk_fds[%d] err:%d errno:%d\n", + i, err, errno); + + if (i == first) { + err = setsockopt(sk_fds[i], SOL_SOCKET, + SO_ATTACH_REUSEPORT_EBPF, + &select_by_skb_data_prog, + sizeof(select_by_skb_data_prog)); + CHECK(err == -1, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", + "err:%d errno:%d\n", err, errno); + } + + err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); + CHECK(err == -1, "bind()", "sk_fds[%d] err:%d errno:%d\n", + i, err, errno); + + if (type == SOCK_STREAM) { + err = listen(sk_fds[i], 10); + CHECK(err == -1, "listen()", + "sk_fds[%d] err:%d errno:%d\n", + i, err, errno); + } + + err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], + BPF_NOEXIST); + CHECK(err == -1, "update_elem(reuseport_array)", + "sk_fds[%d] err:%d errno:%d\n", i, err, errno); + + if (i == first) { + socklen_t addrlen = sizeof(srv_sa); + + err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, + &addrlen); + CHECK(err == -1, "getsockname()", + "sk_fds[%d] err:%d errno:%d\n", i, err, errno); + } + } + + epfd = epoll_create(1); + CHECK(epfd == -1, "epoll_create(1)", + "epfd:%d errno:%d\n", epfd, errno); + + ev.events = EPOLLIN; + for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { + ev.data.u32 = i; + err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); + CHECK(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); + } +} + +static void setup_per_test(int type, unsigned short family, bool inany) +{ + int ovr = -1, err; + + prepare_sk_fds(type, family, inany); + err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, + BPF_ANY); + CHECK(err == -1, "update_elem(tmp_index_ovr_map, 0, -1)", + "err:%d errno:%d\n", err, errno); +} + +static void cleanup_per_test(void) +{ + int i, err; + + for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) + close(sk_fds[i]); + close(epfd); + + err = bpf_map_delete_elem(outer_map, &index_zero); + CHECK(err == -1, "delete_elem(outer_map)", + "err:%d errno:%d\n", err, errno); +} + +static void cleanup(void) +{ + close(outer_map); + close(reuseport_array); + bpf_object__close(obj); +} + +static void test_all(void) +{ + /* Extra SOCK_STREAM to test bind_inany==true */ + const int types[] = { SOCK_STREAM, SOCK_DGRAM, SOCK_STREAM }; + const char * const type_strings[] = { "TCP", "UDP", "TCP" }; + const char * const family_strings[] = { "IPv6", "IPv4" }; + const unsigned short families[] = { AF_INET6, AF_INET }; + const bool bind_inany[] = { false, false, true }; + int t, f, err; + + for (f = 0; f < ARRAY_SIZE(families); f++) { + unsigned short family = families[f]; + + for (t = 0; t < ARRAY_SIZE(types); t++) { + bool inany = bind_inany[t]; + int type = types[t]; + + printf("######## %s/%s %s ########\n", + family_strings[f], type_strings[t], + inany ? " INANY " : "LOOPBACK"); + + setup_per_test(type, family, inany); + + test_err_inner_map(type, family); + + /* Install reuseport_array to the outer_map */ + err = bpf_map_update_elem(outer_map, &index_zero, + &reuseport_array, BPF_ANY); + CHECK(err == -1, "update_elem(outer_map)", + "err:%d errno:%d\n", err, errno); + + test_err_skb_data(type, family); + test_err_sk_select_port(type, family); + test_pass(type, family); + test_syncookie(type, family); + test_pass_on_err(type, family); + + cleanup_per_test(); + printf("\n"); + } + } +} + +int main(int argc, const char **argv) +{ + create_maps(); + prepare_bpf_obj(); + saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL); + saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL); + enable_fastopen(); + disable_syncookie(); + atexit(restore_sysctls); + + test_all(); + + cleanup(); + return 0; +} diff --git a/tools/testing/selftests/bpf/test_select_reuseport_common.h b/tools/testing/selftests/bpf/test_select_reuseport_common.h new file mode 100644 index 000000000000..08eb2a9f145f --- /dev/null +++ b/tools/testing/selftests/bpf/test_select_reuseport_common.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#ifndef __TEST_SELECT_REUSEPORT_COMMON_H +#define __TEST_SELECT_REUSEPORT_COMMON_H + +#include <linux/types.h> + +enum result { + DROP_ERR_INNER_MAP, + DROP_ERR_SKB_DATA, + DROP_ERR_SK_SELECT_REUSEPORT, + DROP_MISC, + PASS, + PASS_ERR_SK_SELECT_REUSEPORT, + NR_RESULTS, +}; + +struct cmd { + __u32 reuseport_index; + __u32 pass_on_failure; +}; + +struct data_check { + __u32 ip_protocol; + __u32 skb_addrs[8]; + __u16 skb_ports[2]; + __u16 eth_protocol; + __u8 bind_inany; + __u8 equal_check_end[0]; + + __u32 len; + __u32 hash; +}; + +#endif diff --git a/tools/testing/selftests/bpf/test_select_reuseport_kern.c b/tools/testing/selftests/bpf/test_select_reuseport_kern.c new file mode 100644 index 000000000000..5b54ec637ada --- /dev/null +++ b/tools/testing/selftests/bpf/test_select_reuseport_kern.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2018 Facebook */ + +#include <stdlib.h> +#include <linux/in.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <linux/bpf.h> +#include <linux/types.h> +#include <linux/if_ether.h> + +#include "bpf_endian.h" +#include "bpf_helpers.h" +#include "test_select_reuseport_common.h" + +int _version SEC("version") = 1; + +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +struct bpf_map_def SEC("maps") outer_map = { + .type = BPF_MAP_TYPE_ARRAY_OF_MAPS, + .key_size = sizeof(__u32), + .value_size = sizeof(__u32), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") result_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u32), + .max_entries = NR_RESULTS, +}; + +struct bpf_map_def SEC("maps") tmp_index_ovr_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(int), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") linum_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u32), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") data_check_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct data_check), + .max_entries = 1, +}; + +#define GOTO_DONE(_result) ({ \ + result = (_result); \ + linum = __LINE__; \ + goto done; \ +}) + +SEC("select_by_skb_data") +int _select_by_skb_data(struct sk_reuseport_md *reuse_md) +{ + __u32 linum, index = 0, flags = 0, index_zero = 0; + __u32 *result_cnt, *linum_value; + struct data_check data_check = {}; + struct cmd *cmd, cmd_copy; + void *data, *data_end; + void *reuseport_array; + enum result result; + int *index_ovr; + int err; + + data = reuse_md->data; + data_end = reuse_md->data_end; + data_check.len = reuse_md->len; + data_check.eth_protocol = reuse_md->eth_protocol; + data_check.ip_protocol = reuse_md->ip_protocol; + data_check.hash = reuse_md->hash; + data_check.bind_inany = reuse_md->bind_inany; + if (data_check.eth_protocol == bpf_htons(ETH_P_IP)) { + if (bpf_skb_load_bytes_relative(reuse_md, + offsetof(struct iphdr, saddr), + data_check.skb_addrs, 8, + BPF_HDR_START_NET)) + GOTO_DONE(DROP_MISC); + } else { + if (bpf_skb_load_bytes_relative(reuse_md, + offsetof(struct ipv6hdr, saddr), + data_check.skb_addrs, 32, + BPF_HDR_START_NET)) + GOTO_DONE(DROP_MISC); + } + + /* + * The ip_protocol could be a compile time decision + * if the bpf_prog.o is dedicated to either TCP or + * UDP. + * + * Otherwise, reuse_md->ip_protocol or + * the protocol field in the iphdr can be used. + */ + if (data_check.ip_protocol == IPPROTO_TCP) { + struct tcphdr *th = data; + + if (th + 1 > data_end) + GOTO_DONE(DROP_MISC); + + data_check.skb_ports[0] = th->source; + data_check.skb_ports[1] = th->dest; + + if ((th->doff << 2) + sizeof(*cmd) > data_check.len) + GOTO_DONE(DROP_ERR_SKB_DATA); + if (bpf_skb_load_bytes(reuse_md, th->doff << 2, &cmd_copy, + sizeof(cmd_copy))) + GOTO_DONE(DROP_MISC); + cmd = &cmd_copy; + } else if (data_check.ip_protocol == IPPROTO_UDP) { + struct udphdr *uh = data; + + if (uh + 1 > data_end) + GOTO_DONE(DROP_MISC); + + data_check.skb_ports[0] = uh->source; + data_check.skb_ports[1] = uh->dest; + + if (sizeof(struct udphdr) + sizeof(*cmd) > data_check.len) + GOTO_DONE(DROP_ERR_SKB_DATA); + if (data + sizeof(struct udphdr) + sizeof(*cmd) > data_end) { + if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), + &cmd_copy, sizeof(cmd_copy))) + GOTO_DONE(DROP_MISC); + cmd = &cmd_copy; + } else { + cmd = data + sizeof(struct udphdr); + } + } else { + GOTO_DONE(DROP_MISC); + } + + reuseport_array = bpf_map_lookup_elem(&outer_map, &index_zero); + if (!reuseport_array) + GOTO_DONE(DROP_ERR_INNER_MAP); + + index = cmd->reuseport_index; + index_ovr = bpf_map_lookup_elem(&tmp_index_ovr_map, &index_zero); + if (!index_ovr) + GOTO_DONE(DROP_MISC); + + if (*index_ovr != -1) { + index = *index_ovr; + *index_ovr = -1; + } + err = bpf_sk_select_reuseport(reuse_md, reuseport_array, &index, + flags); + if (!err) + GOTO_DONE(PASS); + + if (cmd->pass_on_failure) + GOTO_DONE(PASS_ERR_SK_SELECT_REUSEPORT); + else + GOTO_DONE(DROP_ERR_SK_SELECT_REUSEPORT); + +done: + result_cnt = bpf_map_lookup_elem(&result_map, &result); + if (!result_cnt) + return SK_DROP; + + bpf_map_update_elem(&linum_map, &index_zero, &linum, BPF_ANY); + bpf_map_update_elem(&data_check_map, &index_zero, &data_check, BPF_ANY); + + (*result_cnt)++; + return result < PASS ? SK_DROP : SK_PASS; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_skb_cgroup_id.sh b/tools/testing/selftests/bpf/test_skb_cgroup_id.sh new file mode 100755 index 000000000000..42544a969abc --- /dev/null +++ b/tools/testing/selftests/bpf/test_skb_cgroup_id.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2018 Facebook + +set -eu + +wait_for_ip() +{ + local _i + echo -n "Wait for testing link-local IP to become available " + for _i in $(seq ${MAX_PING_TRIES}); do + echo -n "." + if ping -6 -q -c 1 -W 1 ff02::1%${TEST_IF} >/dev/null 2>&1; then + echo " OK" + return + fi + sleep 1 + done + echo 1>&2 "ERROR: Timeout waiting for test IP to become available." + exit 1 +} + +setup() +{ + # Create testing interfaces not to interfere with current environment. + ip link add dev ${TEST_IF} type veth peer name ${TEST_IF_PEER} + ip link set ${TEST_IF} up + ip link set ${TEST_IF_PEER} up + + wait_for_ip + + tc qdisc add dev ${TEST_IF} clsact + tc filter add dev ${TEST_IF} egress bpf obj ${BPF_PROG_OBJ} \ + sec ${BPF_PROG_SECTION} da + + BPF_PROG_ID=$(tc filter show dev ${TEST_IF} egress | \ + awk '/ id / {sub(/.* id /, "", $0); print($1)}') +} + +cleanup() +{ + ip link del ${TEST_IF} 2>/dev/null || : + ip link del ${TEST_IF_PEER} 2>/dev/null || : +} + +main() +{ + trap cleanup EXIT 2 3 6 15 + setup + ${PROG} ${TEST_IF} ${BPF_PROG_ID} +} + +DIR=$(dirname $0) +TEST_IF="test_cgid_1" +TEST_IF_PEER="test_cgid_2" +MAX_PING_TRIES=5 +BPF_PROG_OBJ="${DIR}/test_skb_cgroup_id_kern.o" +BPF_PROG_SECTION="cgroup_id_logger" +BPF_PROG_ID=0 +PROG="${DIR}/test_skb_cgroup_id_user" + +main diff --git a/tools/testing/selftests/bpf/test_skb_cgroup_id_kern.c b/tools/testing/selftests/bpf/test_skb_cgroup_id_kern.c new file mode 100644 index 000000000000..68cf9829f5a7 --- /dev/null +++ b/tools/testing/selftests/bpf/test_skb_cgroup_id_kern.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2018 Facebook + +#include <linux/bpf.h> +#include <linux/pkt_cls.h> + +#include <string.h> + +#include "bpf_helpers.h" + +#define NUM_CGROUP_LEVELS 4 + +struct bpf_map_def SEC("maps") cgroup_ids = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u64), + .max_entries = NUM_CGROUP_LEVELS, +}; + +static __always_inline void log_nth_level(struct __sk_buff *skb, __u32 level) +{ + __u64 id; + + /* [1] &level passed to external function that may change it, it's + * incompatible with loop unroll. + */ + id = bpf_skb_ancestor_cgroup_id(skb, level); + bpf_map_update_elem(&cgroup_ids, &level, &id, 0); +} + +SEC("cgroup_id_logger") +int log_cgroup_id(struct __sk_buff *skb) +{ + /* Loop unroll can't be used here due to [1]. Unrolling manually. + * Number of calls should be in sync with NUM_CGROUP_LEVELS. + */ + log_nth_level(skb, 0); + log_nth_level(skb, 1); + log_nth_level(skb, 2); + log_nth_level(skb, 3); + + return TC_ACT_OK; +} + +int _version SEC("version") = 1; + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_skb_cgroup_id_user.c b/tools/testing/selftests/bpf/test_skb_cgroup_id_user.c new file mode 100644 index 000000000000..c121cc59f314 --- /dev/null +++ b/tools/testing/selftests/bpf/test_skb_cgroup_id_user.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2018 Facebook + +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include <arpa/inet.h> +#include <net/if.h> +#include <netinet/in.h> +#include <sys/socket.h> +#include <sys/types.h> + + +#include <bpf/bpf.h> +#include <bpf/libbpf.h> + +#include "bpf_rlimit.h" +#include "cgroup_helpers.h" + +#define CGROUP_PATH "/skb_cgroup_test" +#define NUM_CGROUP_LEVELS 4 + +/* RFC 4291, Section 2.7.1 */ +#define LINKLOCAL_MULTICAST "ff02::1" + +static int mk_dst_addr(const char *ip, const char *iface, + struct sockaddr_in6 *dst) +{ + memset(dst, 0, sizeof(*dst)); + + dst->sin6_family = AF_INET6; + dst->sin6_port = htons(1025); + + if (inet_pton(AF_INET6, ip, &dst->sin6_addr) != 1) { + log_err("Invalid IPv6: %s", ip); + return -1; + } + + dst->sin6_scope_id = if_nametoindex(iface); + if (!dst->sin6_scope_id) { + log_err("Failed to get index of iface: %s", iface); + return -1; + } + + return 0; +} + +static int send_packet(const char *iface) +{ + struct sockaddr_in6 dst; + char msg[] = "msg"; + int err = 0; + int fd = -1; + + if (mk_dst_addr(LINKLOCAL_MULTICAST, iface, &dst)) + goto err; + + fd = socket(AF_INET6, SOCK_DGRAM, 0); + if (fd == -1) { + log_err("Failed to create UDP socket"); + goto err; + } + + if (sendto(fd, &msg, sizeof(msg), 0, (const struct sockaddr *)&dst, + sizeof(dst)) == -1) { + log_err("Failed to send datagram"); + goto err; + } + + goto out; +err: + err = -1; +out: + if (fd >= 0) + close(fd); + return err; +} + +int get_map_fd_by_prog_id(int prog_id) +{ + struct bpf_prog_info info = {}; + __u32 info_len = sizeof(info); + __u32 map_ids[1]; + int prog_fd = -1; + int map_fd = -1; + + prog_fd = bpf_prog_get_fd_by_id(prog_id); + if (prog_fd < 0) { + log_err("Failed to get fd by prog id %d", prog_id); + goto err; + } + + info.nr_map_ids = 1; + info.map_ids = (__u64) (unsigned long) map_ids; + + if (bpf_obj_get_info_by_fd(prog_fd, &info, &info_len)) { + log_err("Failed to get info by prog fd %d", prog_fd); + goto err; + } + + if (!info.nr_map_ids) { + log_err("No maps found for prog fd %d", prog_fd); + goto err; + } + + map_fd = bpf_map_get_fd_by_id(map_ids[0]); + if (map_fd < 0) + log_err("Failed to get fd by map id %d", map_ids[0]); +err: + if (prog_fd >= 0) + close(prog_fd); + return map_fd; +} + +int check_ancestor_cgroup_ids(int prog_id) +{ + __u64 actual_ids[NUM_CGROUP_LEVELS], expected_ids[NUM_CGROUP_LEVELS]; + __u32 level; + int err = 0; + int map_fd; + + expected_ids[0] = 0x100000001; /* root cgroup */ + expected_ids[1] = get_cgroup_id(""); + expected_ids[2] = get_cgroup_id(CGROUP_PATH); + expected_ids[3] = 0; /* non-existent cgroup */ + + map_fd = get_map_fd_by_prog_id(prog_id); + if (map_fd < 0) + goto err; + + for (level = 0; level < NUM_CGROUP_LEVELS; ++level) { + if (bpf_map_lookup_elem(map_fd, &level, &actual_ids[level])) { + log_err("Failed to lookup key %d", level); + goto err; + } + if (actual_ids[level] != expected_ids[level]) { + log_err("%llx (actual) != %llx (expected), level: %u\n", + actual_ids[level], expected_ids[level], level); + goto err; + } + } + + goto out; +err: + err = -1; +out: + if (map_fd >= 0) + close(map_fd); + return err; +} + +int main(int argc, char **argv) +{ + int cgfd = -1; + int err = 0; + + if (argc < 3) { + fprintf(stderr, "Usage: %s iface prog_id\n", argv[0]); + exit(EXIT_FAILURE); + } + + if (setup_cgroup_environment()) + goto err; + + cgfd = create_and_get_cgroup(CGROUP_PATH); + if (!cgfd) + goto err; + + if (join_cgroup(CGROUP_PATH)) + goto err; + + if (send_packet(argv[1])) + goto err; + + if (check_ancestor_cgroup_ids(atoi(argv[2]))) + goto err; + + goto out; +err: + err = -1; +out: + close(cgfd); + cleanup_cgroup_environment(); + printf("[%s]\n", err ? "FAIL" : "PASS"); + return err; +} diff --git a/tools/testing/selftests/bpf/test_sock.c b/tools/testing/selftests/bpf/test_sock.c index f4d99fabc56d..b8ebe2f58074 100644 --- a/tools/testing/selftests/bpf/test_sock.c +++ b/tools/testing/selftests/bpf/test_sock.c @@ -14,10 +14,7 @@ #include "cgroup_helpers.h" #include "bpf_rlimit.h" - -#ifndef ARRAY_SIZE -# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) -#endif +#include "bpf_util.h" #define CG_PATH "/foo" #define MAX_INSNS 512 diff --git a/tools/testing/selftests/bpf/test_sock_addr.c b/tools/testing/selftests/bpf/test_sock_addr.c index 2e45c92d1111..aeeb76a54d63 100644 --- a/tools/testing/selftests/bpf/test_sock_addr.c +++ b/tools/testing/selftests/bpf/test_sock_addr.c @@ -20,15 +20,12 @@ #include "cgroup_helpers.h" #include "bpf_rlimit.h" +#include "bpf_util.h" #ifndef ENOTSUPP # define ENOTSUPP 524 #endif -#ifndef ARRAY_SIZE -# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) -#endif - #define CG_PATH "/foo" #define CONNECT4_PROG_PATH "./connect4_prog.o" #define CONNECT6_PROG_PATH "./connect6_prog.o" diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index 9e78df207919..0c7d9e556b47 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -354,7 +354,7 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, while (s->bytes_recvd < total_bytes) { if (txmsg_cork) { timeout.tv_sec = 0; - timeout.tv_usec = 1000; + timeout.tv_usec = 300000; } else { timeout.tv_sec = 1; timeout.tv_usec = 0; diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 452cf5c6c784..67c412d19c09 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -42,12 +42,9 @@ #endif #include "bpf_rlimit.h" #include "bpf_rand.h" +#include "bpf_util.h" #include "../../../include/linux/filter.h" -#ifndef ARRAY_SIZE -# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) -#endif - #define MAX_INSNS BPF_MAXINSNS #define MAX_FIXUPS 8 #define MAX_NR_MAPS 8 diff --git a/tools/testing/selftests/tc-testing/README b/tools/testing/selftests/tc-testing/README index 3a0336782d2d..49a6f8c3fdae 100644 --- a/tools/testing/selftests/tc-testing/README +++ b/tools/testing/selftests/tc-testing/README @@ -17,6 +17,10 @@ REQUIREMENTS * The kernel must have veth support available, as a veth pair is created prior to running the tests. +* The kernel must have the appropriate infrastructure enabled to run all tdc + unit tests. See the config file in this directory for minimum required + features. As new tests will be added, config options list will be updated. + * All tc-related features being tested must be built in or available as modules. To check what is required in current setup run: ./tdc.py -c @@ -109,8 +113,8 @@ COMMAND LINE ARGUMENTS Run tdc.py -h to see the full list of available arguments. usage: tdc.py [-h] [-p PATH] [-D DIR [DIR ...]] [-f FILE [FILE ...]] - [-c [CATG [CATG ...]]] [-e ID [ID ...]] [-l] [-s] [-i] [-v] - [-d DEVICE] [-n NS] [-V] + [-c [CATG [CATG ...]]] [-e ID [ID ...]] [-l] [-s] [-i] [-v] [-N] + [-d DEVICE] [-P] [-n] [-V] Linux TC unit tests @@ -118,8 +122,10 @@ optional arguments: -h, --help show this help message and exit -p PATH, --path PATH The full path to the tc executable to use -v, --verbose Show the commands that are being run + -N, --notap Suppress tap results for command under test -d DEVICE, --device DEVICE Execute the test case in flower category + -P, --pause Pause execution just before post-suite stage selection: select which test cases: files plus directories; filtered by categories @@ -146,10 +152,10 @@ action: -i, --id Generate ID numbers for new test cases netns: - options for nsPlugin(run commands in net namespace) + options for nsPlugin (run commands in net namespace) - -n NS, --namespace NS - Run commands in namespace NS + -n, --namespace + Run commands in namespace as specified in tdc_config.py valgrind: options for valgrindPlugin (run command under test under Valgrind) diff --git a/tools/testing/selftests/tc-testing/config b/tools/testing/selftests/tc-testing/config new file mode 100644 index 000000000000..203302065458 --- /dev/null +++ b/tools/testing/selftests/tc-testing/config @@ -0,0 +1,48 @@ +CONFIG_NET_SCHED=y + +# +# Queueing/Scheduling +# +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_INGRESS=m + +# +# Classification +# +CONFIG_NET_CLS=y +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_STACK=32 +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_IPT=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_NET_CLS_IND=y +CONFIG_NET_SCH_FIFO=y |
