diff options
Diffstat (limited to 'arch/riscv/mm')
| -rw-r--r-- | arch/riscv/mm/Makefile | 4 | ||||
| -rw-r--r-- | arch/riscv/mm/cacheflush.c | 27 | ||||
| -rw-r--r-- | arch/riscv/mm/context.c | 2 | ||||
| -rw-r--r-- | arch/riscv/mm/extable.c | 4 | ||||
| -rw-r--r-- | arch/riscv/mm/fault.c | 6 | ||||
| -rw-r--r-- | arch/riscv/mm/init.c | 60 | ||||
| -rw-r--r-- | arch/riscv/mm/ioremap.c | 84 | ||||
| -rw-r--r-- | arch/riscv/mm/sifive_l2_cache.c | 178 | ||||
| -rw-r--r-- | arch/riscv/mm/tlbflush.c | 25 |
9 files changed, 101 insertions, 289 deletions
diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index 9d9a17335686..a1bd95c8047a 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -6,12 +6,10 @@ CFLAGS_REMOVE_init.o = -pg endif obj-y += init.o -obj-y += fault.o obj-y += extable.o -obj-y += ioremap.o +obj-$(CONFIG_MMU) += fault.o obj-y += cacheflush.o obj-y += context.o -obj-y += sifive_l2_cache.o ifeq ($(CONFIG_MMU),y) obj-$(CONFIG_SMP) += tlbflush.o diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index 3f15938dec89..8930ab7278e6 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -10,10 +10,19 @@ #include <asm/sbi.h> +static void ipi_remote_fence_i(void *info) +{ + return local_flush_icache_all(); +} + void flush_icache_all(void) { - sbi_remote_fence_i(NULL); + if (IS_ENABLED(CONFIG_RISCV_SBI)) + sbi_remote_fence_i(NULL); + else + on_each_cpu(ipi_remote_fence_i, NULL, 1); } +EXPORT_SYMBOL(flush_icache_all); /* * Performs an icache flush for the given MM context. RISC-V has no direct @@ -28,7 +37,7 @@ void flush_icache_all(void) void flush_icache_mm(struct mm_struct *mm, bool local) { unsigned int cpu; - cpumask_t others, hmask, *mask; + cpumask_t others, *mask; preempt_disable(); @@ -46,10 +55,7 @@ void flush_icache_mm(struct mm_struct *mm, bool local) */ cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); local |= cpumask_empty(&others); - if (mm != current->active_mm || !local) { - riscv_cpuid_to_hartid_mask(&others, &hmask); - sbi_remote_fence_i(hmask.bits); - } else { + if (mm == current->active_mm && local) { /* * It's assumed that at least one strongly ordered operation is * performed on this hart between setting a hart's cpumask bit @@ -59,6 +65,13 @@ void flush_icache_mm(struct mm_struct *mm, bool local) * with flush_icache_deferred(). */ smp_mb(); + } else if (IS_ENABLED(CONFIG_RISCV_SBI)) { + cpumask_t hartid_mask; + + riscv_cpuid_to_hartid_mask(&others, &hartid_mask); + sbi_remote_fence_i(cpumask_bits(&hartid_mask)); + } else { + on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); } preempt_enable(); @@ -66,6 +79,7 @@ void flush_icache_mm(struct mm_struct *mm, bool local) #endif /* CONFIG_SMP */ +#ifdef CONFIG_MMU void flush_icache_pte(pte_t pte) { struct page *page = pte_page(pte); @@ -73,3 +87,4 @@ void flush_icache_pte(pte_t pte) if (!test_and_set_bit(PG_dcache_clean, &page->flags)) flush_icache_all(); } +#endif /* CONFIG_MMU */ diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index ca66d44156b6..613ec81a8979 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -58,8 +58,10 @@ void switch_mm(struct mm_struct *prev, struct mm_struct *next, cpumask_clear_cpu(cpu, mm_cpumask(prev)); cpumask_set_cpu(cpu, mm_cpumask(next)); +#ifdef CONFIG_MMU csr_write(CSR_SATP, virt_to_pfn(next->pgd) | SATP_MODE); local_flush_tlb_all(); +#endif flush_icache_deferred(next); } diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index 7aed9178d365..2fc729422151 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -15,9 +15,9 @@ int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; - fixup = search_exception_tables(regs->sepc); + fixup = search_exception_tables(regs->epc); if (fixup) { - regs->sepc = fixup->fixup; + regs->epc = fixup->fixup; return 1; } return 0; diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 247b8c859c44..cf7248e07f43 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -34,8 +34,8 @@ asmlinkage void do_page_fault(struct pt_regs *regs) int code = SEGV_MAPERR; vm_fault_t fault; - cause = regs->scause; - addr = regs->sbadaddr; + cause = regs->cause; + addr = regs->badaddr; tsk = current; mm = tsk->mm; @@ -53,7 +53,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) goto vmalloc_fault; /* Enable interrupts if they were enabled in the parent context. */ - if (likely(regs->sstatus & SR_SPIE)) + if (likely(regs->status & SR_PIE)) local_irq_enable(); /* diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 573463d1c799..69f6678db7f3 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -26,6 +26,7 @@ unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] EXPORT_SYMBOL(empty_zero_page); extern char _start[]; +void *dtb_early_va; static void __init zone_sizes_init(void) { @@ -40,11 +41,42 @@ static void __init zone_sizes_init(void) free_area_init_nodes(max_zone_pfns); } -void setup_zero_page(void) +static void setup_zero_page(void) { memset((void *)empty_zero_page, 0, PAGE_SIZE); } +#ifdef CONFIG_DEBUG_VM +static inline void print_mlk(char *name, unsigned long b, unsigned long t) +{ + pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, + (((t) - (b)) >> 10)); +} + +static inline void print_mlm(char *name, unsigned long b, unsigned long t) +{ + pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, + (((t) - (b)) >> 20)); +} + +static void print_vm_layout(void) +{ + pr_notice("Virtual kernel memory layout:\n"); + print_mlk("fixmap", (unsigned long)FIXADDR_START, + (unsigned long)FIXADDR_TOP); + print_mlm("pci io", (unsigned long)PCI_IO_START, + (unsigned long)PCI_IO_END); + print_mlm("vmemmap", (unsigned long)VMEMMAP_START, + (unsigned long)VMEMMAP_END); + print_mlm("vmalloc", (unsigned long)VMALLOC_START, + (unsigned long)VMALLOC_END); + print_mlm("lowmem", (unsigned long)PAGE_OFFSET, + (unsigned long)high_memory); +} +#else +static void print_vm_layout(void) { } +#endif /* CONFIG_DEBUG_VM */ + void __init mem_init(void) { #ifdef CONFIG_FLATMEM @@ -55,6 +87,7 @@ void __init mem_init(void) memblock_free_all(); mem_init_print_info(NULL); + print_vm_layout(); } #ifdef CONFIG_BLK_DEV_INITRD @@ -142,12 +175,12 @@ void __init setup_bootmem(void) } } +#ifdef CONFIG_MMU unsigned long va_pa_offset; EXPORT_SYMBOL(va_pa_offset); unsigned long pfn_base; EXPORT_SYMBOL(pfn_base); -void *dtb_early_va; pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; @@ -273,7 +306,6 @@ static void __init create_pmd_mapping(pmd_t *pmdp, #define get_pgd_next_virt(__pa) get_pmd_virt(__pa) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) -#define PTE_PARENT_SIZE PMD_SIZE #define fixmap_pgd_next fixmap_pmd #else #define pgd_next_t pte_t @@ -281,7 +313,6 @@ static void __init create_pmd_mapping(pmd_t *pmdp, #define get_pgd_next_virt(__pa) get_pte_virt(__pa) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ create_pte_mapping(__nextp, __va, __pa, __sz, __prot) -#define PTE_PARENT_SIZE PGDIR_SIZE #define fixmap_pgd_next fixmap_pte #endif @@ -314,14 +345,11 @@ static void __init create_pgd_mapping(pgd_t *pgdp, static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) { - uintptr_t map_size = PAGE_SIZE; - - /* Upgrade to PMD/PGDIR mappings whenever possible */ - if (!(base & (PTE_PARENT_SIZE - 1)) && - !(size & (PTE_PARENT_SIZE - 1))) - map_size = PTE_PARENT_SIZE; + /* Upgrade to PMD_SIZE mappings whenever possible */ + if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) + return PAGE_SIZE; - return map_size; + return PMD_SIZE; } /* @@ -449,6 +477,16 @@ static void __init setup_vm_final(void) csr_write(CSR_SATP, PFN_DOWN(__pa(swapper_pg_dir)) | SATP_MODE); local_flush_tlb_all(); } +#else +asmlinkage void __init setup_vm(uintptr_t dtb_pa) +{ + dtb_early_va = (void *)dtb_pa; +} + +static inline void setup_vm_final(void) +{ +} +#endif /* CONFIG_MMU */ void __init paging_init(void) { diff --git a/arch/riscv/mm/ioremap.c b/arch/riscv/mm/ioremap.c deleted file mode 100644 index ac621ddb45c0..000000000000 --- a/arch/riscv/mm/ioremap.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * (C) Copyright 1995 1996 Linus Torvalds - * (C) Copyright 2012 Regents of the University of California - */ - -#include <linux/export.h> -#include <linux/mm.h> -#include <linux/vmalloc.h> -#include <linux/io.h> - -#include <asm/pgtable.h> - -/* - * Remap an arbitrary physical address space into the kernel virtual - * address space. Needed when the kernel wants to access high addresses - * directly. - * - * NOTE! We need to allow non-page-aligned mappings too: we will obviously - * have to convert them into an offset in a page-aligned mapping, but the - * caller shouldn't need to know that small detail. - */ -static void __iomem *__ioremap_caller(phys_addr_t addr, size_t size, - pgprot_t prot, void *caller) -{ - phys_addr_t last_addr; - unsigned long offset, vaddr; - struct vm_struct *area; - - /* Disallow wrap-around or zero size */ - last_addr = addr + size - 1; - if (!size || last_addr < addr) - return NULL; - - /* Page-align mappings */ - offset = addr & (~PAGE_MASK); - addr -= offset; - size = PAGE_ALIGN(size + offset); - - area = get_vm_area_caller(size, VM_IOREMAP, caller); - if (!area) - return NULL; - vaddr = (unsigned long)area->addr; - - if (ioremap_page_range(vaddr, vaddr + size, addr, prot)) { - free_vm_area(area); - return NULL; - } - - return (void __iomem *)(vaddr + offset); -} - -/* - * ioremap - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - * - * ioremap performs a platform specific sequence of operations to - * make bus memory CPU accessible via the readb/readw/readl/writeb/ - * writew/writel functions and the other mmio helpers. The returned - * address is not guaranteed to be usable directly as a virtual - * address. - * - * Must be freed with iounmap. - */ -void __iomem *ioremap(phys_addr_t offset, unsigned long size) -{ - return __ioremap_caller(offset, size, PAGE_KERNEL, - __builtin_return_address(0)); -} -EXPORT_SYMBOL(ioremap); - - -/** - * iounmap - Free a IO remapping - * @addr: virtual address from ioremap_* - * - * Caller must ensure there is only one unmapping for the same pointer. - */ -void iounmap(volatile void __iomem *addr) -{ - vunmap((void *)((unsigned long)addr & PAGE_MASK)); -} -EXPORT_SYMBOL(iounmap); diff --git a/arch/riscv/mm/sifive_l2_cache.c b/arch/riscv/mm/sifive_l2_cache.c deleted file mode 100644 index a9ffff3277c7..000000000000 --- a/arch/riscv/mm/sifive_l2_cache.c +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * SiFive L2 cache controller Driver - * - * Copyright (C) 2018-2019 SiFive, Inc. - * - */ -#include <linux/debugfs.h> -#include <linux/interrupt.h> -#include <linux/of_irq.h> -#include <linux/of_address.h> -#include <asm/sifive_l2_cache.h> - -#define SIFIVE_L2_DIRECCFIX_LOW 0x100 -#define SIFIVE_L2_DIRECCFIX_HIGH 0x104 -#define SIFIVE_L2_DIRECCFIX_COUNT 0x108 - -#define SIFIVE_L2_DATECCFIX_LOW 0x140 -#define SIFIVE_L2_DATECCFIX_HIGH 0x144 -#define SIFIVE_L2_DATECCFIX_COUNT 0x148 - -#define SIFIVE_L2_DATECCFAIL_LOW 0x160 -#define SIFIVE_L2_DATECCFAIL_HIGH 0x164 -#define SIFIVE_L2_DATECCFAIL_COUNT 0x168 - -#define SIFIVE_L2_CONFIG 0x00 -#define SIFIVE_L2_WAYENABLE 0x08 -#define SIFIVE_L2_ECCINJECTERR 0x40 - -#define SIFIVE_L2_MAX_ECCINTR 3 - -static void __iomem *l2_base; -static int g_irq[SIFIVE_L2_MAX_ECCINTR]; - -enum { - DIR_CORR = 0, - DATA_CORR, - DATA_UNCORR, -}; - -#ifdef CONFIG_DEBUG_FS -static struct dentry *sifive_test; - -static ssize_t l2_write(struct file *file, const char __user *data, - size_t count, loff_t *ppos) -{ - unsigned int val; - - if (kstrtouint_from_user(data, count, 0, &val)) - return -EINVAL; - if ((val >= 0 && val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) - writel(val, l2_base + SIFIVE_L2_ECCINJECTERR); - else - return -EINVAL; - return count; -} - -static const struct file_operations l2_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = l2_write -}; - -static void setup_sifive_debug(void) -{ - sifive_test = debugfs_create_dir("sifive_l2_cache", NULL); - - debugfs_create_file("sifive_debug_inject_error", 0200, - sifive_test, NULL, &l2_fops); -} -#endif - -static void l2_config_read(void) -{ - u32 regval, val; - - regval = readl(l2_base + SIFIVE_L2_CONFIG); - val = regval & 0xFF; - pr_info("L2CACHE: No. of Banks in the cache: %d\n", val); - val = (regval & 0xFF00) >> 8; - pr_info("L2CACHE: No. of ways per bank: %d\n", val); - val = (regval & 0xFF0000) >> 16; - pr_info("L2CACHE: Sets per bank: %llu\n", (uint64_t)1 << val); - val = (regval & 0xFF000000) >> 24; - pr_info("L2CACHE: Bytes per cache block: %llu\n", (uint64_t)1 << val); - - regval = readl(l2_base + SIFIVE_L2_WAYENABLE); - pr_info("L2CACHE: Index of the largest way enabled: %d\n", regval); -} - -static const struct of_device_id sifive_l2_ids[] = { - { .compatible = "sifive,fu540-c000-ccache" }, - { /* end of table */ }, -}; - -static ATOMIC_NOTIFIER_HEAD(l2_err_chain); - -int register_sifive_l2_error_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_register(&l2_err_chain, nb); -} -EXPORT_SYMBOL_GPL(register_sifive_l2_error_notifier); - -int unregister_sifive_l2_error_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_unregister(&l2_err_chain, nb); -} -EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier); - -static irqreturn_t l2_int_handler(int irq, void *device) -{ - unsigned int add_h, add_l; - - if (irq == g_irq[DIR_CORR]) { - add_h = readl(l2_base + SIFIVE_L2_DIRECCFIX_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DIRECCFIX_LOW); - pr_err("L2CACHE: DirError @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DirError interrupt sig */ - readl(l2_base + SIFIVE_L2_DIRECCFIX_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE, - "DirECCFix"); - } - if (irq == g_irq[DATA_CORR]) { - add_h = readl(l2_base + SIFIVE_L2_DATECCFIX_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DATECCFIX_LOW); - pr_err("L2CACHE: DataError @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DataError interrupt sig */ - readl(l2_base + SIFIVE_L2_DATECCFIX_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE, - "DatECCFix"); - } - if (irq == g_irq[DATA_UNCORR]) { - add_h = readl(l2_base + SIFIVE_L2_DATECCFAIL_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DATECCFAIL_LOW); - pr_err("L2CACHE: DataFail @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DataFail interrupt sig */ - readl(l2_base + SIFIVE_L2_DATECCFAIL_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_UE, - "DatECCFail"); - } - - return IRQ_HANDLED; -} - -static int __init sifive_l2_init(void) -{ - struct device_node *np; - struct resource res; - int i, rc; - - np = of_find_matching_node(NULL, sifive_l2_ids); - if (!np) - return -ENODEV; - - if (of_address_to_resource(np, 0, &res)) - return -ENODEV; - - l2_base = ioremap(res.start, resource_size(&res)); - if (!l2_base) - return -ENOMEM; - - for (i = 0; i < SIFIVE_L2_MAX_ECCINTR; i++) { - g_irq[i] = irq_of_parse_and_map(np, i); - rc = request_irq(g_irq[i], l2_int_handler, 0, "l2_ecc", NULL); - if (rc) { - pr_err("L2CACHE: Could not request IRQ %d\n", g_irq[i]); - return rc; - } - } - - l2_config_read(); - -#ifdef CONFIG_DEBUG_FS - setup_sifive_debug(); -#endif - return 0; -} -device_initcall(sifive_l2_init); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 24cd33d2c48f..720b443c4528 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -2,6 +2,7 @@ #include <linux/mm.h> #include <linux/smp.h> +#include <linux/sched.h> #include <asm/sbi.h> void flush_tlb_all(void) @@ -9,13 +10,33 @@ void flush_tlb_all(void) sbi_remote_sfence_vma(NULL, 0, -1); } +/* + * This function must not be called with cmask being null. + * Kernel may panic if cmask is NULL. + */ static void __sbi_tlb_flush_range(struct cpumask *cmask, unsigned long start, unsigned long size) { struct cpumask hmask; + unsigned int cpuid; - riscv_cpuid_to_hartid_mask(cmask, &hmask); - sbi_remote_sfence_vma(hmask.bits, start, size); + if (cpumask_empty(cmask)) + return; + + cpuid = get_cpu(); + + if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) { + /* local cpu is the only cpu present in cpumask */ + if (size <= PAGE_SIZE) + local_flush_tlb_page(start); + else + local_flush_tlb_all(); + } else { + riscv_cpuid_to_hartid_mask(cmask, &hmask); + sbi_remote_sfence_vma(cpumask_bits(&hmask), start, size); + } + + put_cpu(); } void flush_tlb_mm(struct mm_struct *mm) |
