diff options
Diffstat (limited to 'drivers/iommu')
-rw-r--r-- | drivers/iommu/amd_iommu.c | 8 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_init.c | 101 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_proto.h | 8 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_types.h | 3 | ||||
-rw-r--r-- | drivers/iommu/arm-smmu-v3.c | 2 | ||||
-rw-r--r-- | drivers/iommu/arm-smmu.c | 2 | ||||
-rw-r--r-- | drivers/iommu/exynos-iommu.c | 8 | ||||
-rw-r--r-- | drivers/iommu/intel-iommu.c | 4 | ||||
-rw-r--r-- | drivers/iommu/io-pgtable-arm-v7s.c | 6 | ||||
-rw-r--r-- | drivers/iommu/io-pgtable-arm.c | 6 | ||||
-rw-r--r-- | drivers/iommu/iommu.c | 5 |
11 files changed, 90 insertions, 63 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 98940d1392cb..63cacf5d6cf2 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1234,7 +1234,7 @@ static void __domain_flush_pages(struct protection_domain *domain, build_inv_iommu_pages(&cmd, address, size, domain->id, pde); - for (i = 0; i < amd_iommus_present; ++i) { + for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { if (!domain->dev_iommu[i]) continue; @@ -1278,7 +1278,7 @@ static void domain_flush_complete(struct protection_domain *domain) { int i; - for (i = 0; i < amd_iommus_present; ++i) { + for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { if (domain && !domain->dev_iommu[i]) continue; @@ -3202,7 +3202,7 @@ static void amd_iommu_get_resv_regions(struct device *dev, region = iommu_alloc_resv_region(MSI_RANGE_START, MSI_RANGE_END - MSI_RANGE_START + 1, - 0, IOMMU_RESV_RESERVED); + 0, IOMMU_RESV_MSI); if (!region) return; list_add_tail(®ion->list, head); @@ -3363,7 +3363,7 @@ static int __flush_pasid(struct protection_domain *domain, int pasid, * IOMMU TLB needs to be flushed before Device TLB to * prevent device TLB refill from IOMMU TLB */ - for (i = 0; i < amd_iommus_present; ++i) { + for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { if (domain->dev_iommu[i] == 0) continue; diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 6130278c5d71..5a11328f4d98 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -167,7 +167,9 @@ LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the /* Array to assign indices to IOMMUs*/ struct amd_iommu *amd_iommus[MAX_IOMMUS]; -int amd_iommus_present; + +/* Number of IOMMUs present in the system */ +static int amd_iommus_present; /* IOMMUs have a non-present cache? */ bool amd_iommu_np_cache __read_mostly; @@ -254,10 +256,6 @@ static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); static void init_device_table_dma(void); -static int iommu_pc_get_set_reg_val(struct amd_iommu *iommu, - u8 bank, u8 cntr, u8 fxn, - u64 *value, bool is_write); - static inline void update_last_devid(u16 devid) { if (devid > amd_iommu_last_bdf) @@ -272,6 +270,11 @@ static inline unsigned long tbl_size(int entry_size) return 1UL << shift; } +int amd_iommu_get_num_iommus(void) +{ + return amd_iommus_present; +} + /* Access to l1 and l2 indexed register spaces */ static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) @@ -1336,7 +1339,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); - iommu->index = amd_iommus_present++; + iommu->index = amd_iommus_present++; if (unlikely(iommu->index >= MAX_IOMMUS)) { WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n"); @@ -1477,6 +1480,8 @@ static int __init init_iommu_all(struct acpi_table_header *table) return 0; } +static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, + u8 fxn, u64 *value, bool is_write); static void init_iommu_perf_ctr(struct amd_iommu *iommu) { @@ -1488,8 +1493,8 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu) amd_iommu_pc_present = true; /* Check if the performance counters can be written to */ - if ((0 != iommu_pc_get_set_reg_val(iommu, 0, 0, 0, &val, true)) || - (0 != iommu_pc_get_set_reg_val(iommu, 0, 0, 0, &val2, false)) || + if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) || + (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) || (val != val2)) { pr_err("AMD-Vi: Unable to write to IOMMU perf counter.\n"); amd_iommu_pc_present = false; @@ -2711,6 +2716,18 @@ bool amd_iommu_v2_supported(void) } EXPORT_SYMBOL(amd_iommu_v2_supported); +struct amd_iommu *get_amd_iommu(unsigned int idx) +{ + unsigned int i = 0; + struct amd_iommu *iommu; + + for_each_iommu(iommu) + if (i++ == idx) + return iommu; + return NULL; +} +EXPORT_SYMBOL(get_amd_iommu); + /**************************************************************************** * * IOMMU EFR Performance Counter support functionality. This code allows @@ -2718,17 +2735,14 @@ EXPORT_SYMBOL(amd_iommu_v2_supported); * ****************************************************************************/ -u8 amd_iommu_pc_get_max_banks(u16 devid) +u8 amd_iommu_pc_get_max_banks(unsigned int idx) { - struct amd_iommu *iommu; - u8 ret = 0; + struct amd_iommu *iommu = get_amd_iommu(idx); - /* locate the iommu governing the devid */ - iommu = amd_iommu_rlookup_table[devid]; if (iommu) - ret = iommu->max_banks; + return iommu->max_banks; - return ret; + return 0; } EXPORT_SYMBOL(amd_iommu_pc_get_max_banks); @@ -2738,62 +2752,69 @@ bool amd_iommu_pc_supported(void) } EXPORT_SYMBOL(amd_iommu_pc_supported); -u8 amd_iommu_pc_get_max_counters(u16 devid) +u8 amd_iommu_pc_get_max_counters(unsigned int idx) { - struct amd_iommu *iommu; - u8 ret = 0; + struct amd_iommu *iommu = get_amd_iommu(idx); - /* locate the iommu governing the devid */ - iommu = amd_iommu_rlookup_table[devid]; if (iommu) - ret = iommu->max_counters; + return iommu->max_counters; - return ret; + return 0; } EXPORT_SYMBOL(amd_iommu_pc_get_max_counters); -static int iommu_pc_get_set_reg_val(struct amd_iommu *iommu, - u8 bank, u8 cntr, u8 fxn, - u64 *value, bool is_write) +static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, + u8 fxn, u64 *value, bool is_write) { u32 offset; u32 max_offset_lim; + /* Make sure the IOMMU PC resource is available */ + if (!amd_iommu_pc_present) + return -ENODEV; + /* Check for valid iommu and pc register indexing */ - if (WARN_ON((fxn > 0x28) || (fxn & 7))) + if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7))) return -ENODEV; - offset = (u32)(((0x40|bank) << 12) | (cntr << 8) | fxn); + offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn); /* Limit the offset to the hw defined mmio region aperture */ - max_offset_lim = (u32)(((0x40|iommu->max_banks) << 12) | + max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) | (iommu->max_counters << 8) | 0x28); if ((offset < MMIO_CNTR_REG_OFFSET) || (offset > max_offset_lim)) return -EINVAL; if (is_write) { - writel((u32)*value, iommu->mmio_base + offset); - writel((*value >> 32), iommu->mmio_base + offset + 4); + u64 val = *value & GENMASK_ULL(47, 0); + + writel((u32)val, iommu->mmio_base + offset); + writel((val >> 32), iommu->mmio_base + offset + 4); } else { *value = readl(iommu->mmio_base + offset + 4); *value <<= 32; - *value = readl(iommu->mmio_base + offset); + *value |= readl(iommu->mmio_base + offset); + *value &= GENMASK_ULL(47, 0); } return 0; } -EXPORT_SYMBOL(amd_iommu_pc_get_set_reg_val); -int amd_iommu_pc_get_set_reg_val(u16 devid, u8 bank, u8 cntr, u8 fxn, - u64 *value, bool is_write) +int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) { - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; + if (!iommu) + return -EINVAL; - /* Make sure the IOMMU PC resource is available */ - if (!amd_iommu_pc_present || iommu == NULL) - return -ENODEV; + return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false); +} +EXPORT_SYMBOL(amd_iommu_pc_get_reg); + +int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) +{ + if (!iommu) + return -EINVAL; - return iommu_pc_get_set_reg_val(iommu, bank, cntr, fxn, - value, is_write); + return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true); } +EXPORT_SYMBOL(amd_iommu_pc_set_reg); diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h index 7eb60c15c582..466260f8a1df 100644 --- a/drivers/iommu/amd_iommu_proto.h +++ b/drivers/iommu/amd_iommu_proto.h @@ -21,6 +21,7 @@ #include "amd_iommu_types.h" +extern int amd_iommu_get_num_iommus(void); extern int amd_iommu_init_dma_ops(void); extern int amd_iommu_init_passthrough(void); extern irqreturn_t amd_iommu_int_thread(int irq, void *data); @@ -56,13 +57,6 @@ extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid, extern int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid); extern struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev); -/* IOMMU Performance Counter functions */ -extern bool amd_iommu_pc_supported(void); -extern u8 amd_iommu_pc_get_max_banks(u16 devid); -extern u8 amd_iommu_pc_get_max_counters(u16 devid); -extern int amd_iommu_pc_get_set_reg_val(u16 devid, u8 bank, u8 cntr, u8 fxn, - u64 *value, bool is_write); - #ifdef CONFIG_IRQ_REMAP extern int amd_iommu_create_irq_domain(struct amd_iommu *iommu); #else diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 003f3ceb2661..4de8f4160bb8 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -611,9 +611,6 @@ extern struct list_head amd_iommu_list; */ extern struct amd_iommu *amd_iommus[MAX_IOMMUS]; -/* Number of IOMMUs present in the system */ -extern int amd_iommus_present; - /* * Declarations for the global list of all protection domains */ diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 5806a6acc94e..591bb96047c9 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1888,7 +1888,7 @@ static void arm_smmu_get_resv_regions(struct device *dev, int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, - prot, IOMMU_RESV_MSI); + prot, IOMMU_RESV_SW_MSI); if (!region) return; diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index abf6496843a6..b493c99e17f7 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1608,7 +1608,7 @@ static void arm_smmu_get_resv_regions(struct device *dev, int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, - prot, IOMMU_RESV_MSI); + prot, IOMMU_RESV_SW_MSI); if (!region) return; diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index a7e0821c9967..c01bfcdb2383 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -512,7 +512,13 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, spin_lock_irqsave(&data->lock, flags); if (data->active && data->version >= MAKE_MMU_VER(3, 3)) { clk_enable(data->clk_master); - __sysmmu_tlb_invalidate_entry(data, iova, 1); + if (sysmmu_block(data)) { + if (data->version >= MAKE_MMU_VER(5, 0)) + __sysmmu_tlb_invalidate(data); + else + __sysmmu_tlb_invalidate_entry(data, iova, 1); + sysmmu_unblock(data); + } clk_disable(data->clk_master); } spin_unlock_irqrestore(&data->lock, flags); diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 238ad3447712..d412a313a372 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -916,7 +916,7 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf * which we used for the IOMMU lookup. Strictly speaking * we could do this for all PCI devices; we only need to * get the BDF# from the scope table for ACPI matches. */ - if (pdev->is_virtfn) + if (pdev && pdev->is_virtfn) goto got_pdev; *bus = drhd->devices[i].bus; @@ -5249,7 +5249,7 @@ static void intel_iommu_get_resv_regions(struct device *device, reg = iommu_alloc_resv_region(IOAPIC_RANGE_START, IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1, - 0, IOMMU_RESV_RESERVED); + 0, IOMMU_RESV_MSI); if (!reg) return; list_add_tail(®->list, head); diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 1c049e2e12bf..8d6ca28c3e1f 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -422,8 +422,12 @@ static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, pte |= ARM_V7S_ATTR_NS_TABLE; __arm_v7s_set_pte(ptep, pte, 1, cfg); - } else { + } else if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) { cptep = iopte_deref(pte, lvl); + } else { + /* We require an unmap first */ + WARN_ON(!selftest_running); + return -EEXIST; } /* Rinse, repeat */ diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index feacc54bec68..f9bc6ebb8140 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -335,8 +335,12 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NSTABLE; __arm_lpae_set_pte(ptep, pte, cfg); - } else { + } else if (!iopte_leaf(pte, lvl)) { cptep = iopte_deref(pte, data); + } else { + /* We require an unmap first */ + WARN_ON(!selftest_running); + return -EEXIST; } /* Rinse, repeat */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 8ea14f41a979..3b67144dead2 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -72,6 +72,7 @@ static const char * const iommu_group_resv_type_string[] = { [IOMMU_RESV_DIRECT] = "direct", [IOMMU_RESV_RESERVED] = "reserved", [IOMMU_RESV_MSI] = "msi", + [IOMMU_RESV_SW_MSI] = "msi", }; #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ @@ -1743,8 +1744,8 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) } struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, - size_t length, - int prot, int type) + size_t length, int prot, + enum iommu_resv_type type) { struct iommu_resv_region *region; |