diff options
Diffstat (limited to 'arch/arm/mm')
29 files changed, 656 insertions, 354 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 9357ff52c221..b169e580bf82 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -662,6 +662,7 @@ config ARM_LPAE bool "Support for the Large Physical Address Extension" depends on MMU && CPU_32v7 && !CPU_32v6 && !CPU_32v5 && \ !CPU_32v4 && !CPU_32v3 + select PHYS_ADDR_T_64BIT help Say Y if you have an ARMv7 processor supporting the LPAE page table format and you would like to access memory beyond the @@ -674,12 +675,6 @@ config ARM_PV_FIXUP def_bool y depends on ARM_LPAE && ARM_PATCH_PHYS_VIRT && ARCH_KEYSTONE -config ARCH_PHYS_ADDR_T_64BIT - def_bool ARM_LPAE - -config ARCH_DMA_ADDR_T_64BIT - bool - config ARM_THUMB bool "Support Thumb user binaries" if !CPU_THUMBONLY && EXPERT depends on CPU_THUMB_CAPABLE @@ -707,7 +702,6 @@ config ARM_THUMBEE config ARM_VIRT_EXT bool - depends on MMU default y if CPU_V7 help Enable the kernel to make use of the ARM Virtualization diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index a0c40610210c..7cb1699fbfc4 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -10,7 +10,7 @@ obj-$(CONFIG_MMU) += fault-armv.o flush.o idmap.o ioremap.o \ ifneq ($(CONFIG_MMU),y) obj-y += nommu.o -obj-$(CONFIG_ARM_MPU) += pmsa-v7.o +obj-$(CONFIG_ARM_MPU) += pmsa-v7.o pmsa-v8.o endif obj-$(CONFIG_ARM_PTDUMP_CORE) += dump.o diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index 2c96190e018b..b54f8f8def36 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -948,14 +948,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) goto fixup; if (ai_usermode & UM_SIGNAL) { - siginfo_t si; - - si.si_signo = SIGBUS; - si.si_errno = 0; - si.si_code = BUS_ADRALN; - si.si_addr = (void __user *)addr; - - force_sig_info(si.si_signo, &si, current); + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)addr, current); } else { /* * We're about to disable the alignment trap and return to diff --git a/arch/arm/mm/cache-b15-rac.c b/arch/arm/mm/cache-b15-rac.c index d9586ba2e63c..c6ed14840c3c 100644 --- a/arch/arm/mm/cache-b15-rac.c +++ b/arch/arm/mm/cache-b15-rac.c @@ -33,7 +33,10 @@ extern void v7_flush_kern_cache_all(void); #define RAC_CPU_SHIFT (8) #define RACCFG_MASK (0xff) #define RAC_CONFIG1_REG (0x7c) -#define RAC_FLUSH_REG (0x80) +/* Brahma-B15 is a quad-core only design */ +#define B15_RAC_FLUSH_REG (0x80) +/* Brahma-B53 is an octo-core design */ +#define B53_RAC_FLUSH_REG (0x84) #define FLUSH_RAC (1 << 0) /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */ @@ -52,6 +55,7 @@ static void __iomem *b15_rac_base; static DEFINE_SPINLOCK(rac_lock); static u32 rac_config0_reg; +static u32 rac_flush_offset; /* Initialization flag to avoid checking for b15_rac_base, and to prevent * multi-platform kernels from crashing here as well. @@ -70,14 +74,14 @@ static inline void __b15_rac_flush(void) { u32 reg; - __raw_writel(FLUSH_RAC, b15_rac_base + RAC_FLUSH_REG); + __raw_writel(FLUSH_RAC, b15_rac_base + rac_flush_offset); do { /* This dmb() is required to force the Bus Interface Unit * to clean oustanding writes, and forces an idle cycle * to be inserted. */ dmb(); - reg = __raw_readl(b15_rac_base + RAC_FLUSH_REG); + reg = __raw_readl(b15_rac_base + rac_flush_offset); } while (reg & FLUSH_RAC); } @@ -287,7 +291,7 @@ static struct syscore_ops b15_rac_syscore_ops = { static int __init b15_rac_init(void) { - struct device_node *dn; + struct device_node *dn, *cpu_dn; int ret = 0, cpu; u32 reg, en_mask = 0; @@ -305,6 +309,24 @@ static int __init b15_rac_init(void) goto out; } + cpu_dn = of_get_cpu_node(0, NULL); + if (!cpu_dn) { + ret = -ENODEV; + goto out; + } + + if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15")) + rac_flush_offset = B15_RAC_FLUSH_REG; + else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53")) + rac_flush_offset = B53_RAC_FLUSH_REG; + else { + pr_err("Unsupported CPU\n"); + of_node_put(cpu_dn); + ret = -EINVAL; + goto out; + } + of_node_put(cpu_dn); + ret = register_reboot_notifier(&b15_rac_reboot_nb); if (ret) { pr_err("failed to register reboot notifier\n"); diff --git a/arch/arm/mm/cache-l2x0-pmu.c b/arch/arm/mm/cache-l2x0-pmu.c index 0a1e2280141f..afe5b4c7b164 100644 --- a/arch/arm/mm/cache-l2x0-pmu.c +++ b/arch/arm/mm/cache-l2x0-pmu.c @@ -293,7 +293,7 @@ static bool l2x0_pmu_group_is_valid(struct perf_event *event) else if (!is_software_event(leader)) return false; - list_for_each_entry(sibling, &leader->sibling_list, group_entry) { + for_each_sibling_event(sibling, leader) { if (sibling->pmu == pmu) num_hw++; else if (!is_software_event(sibling)) diff --git a/arch/arm/mm/copypage-fa.c b/arch/arm/mm/copypage-fa.c index d130a5ece5d5..bf24690ec83a 100644 --- a/arch/arm/mm/copypage-fa.c +++ b/arch/arm/mm/copypage-fa.c @@ -17,26 +17,25 @@ /* * Faraday optimised copy_user_page */ -static void __naked -fa_copy_user_page(void *kto, const void *kfrom) +static void fa_copy_user_page(void *kto, const void *kfrom) { - asm("\ - stmfd sp!, {r4, lr} @ 2\n\ - mov r2, %0 @ 1\n\ -1: ldmia r1!, {r3, r4, ip, lr} @ 4\n\ - stmia r0, {r3, r4, ip, lr} @ 4\n\ - mcr p15, 0, r0, c7, c14, 1 @ 1 clean and invalidate D line\n\ - add r0, r0, #16 @ 1\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4\n\ - stmia r0, {r3, r4, ip, lr} @ 4\n\ - mcr p15, 0, r0, c7, c14, 1 @ 1 clean and invalidate D line\n\ - add r0, r0, #16 @ 1\n\ - subs r2, r2, #1 @ 1\n\ + int tmp; + + asm volatile ("\ +1: ldmia %1!, {r3, r4, ip, lr} @ 4\n\ + stmia %0, {r3, r4, ip, lr} @ 4\n\ + mcr p15, 0, %0, c7, c14, 1 @ 1 clean and invalidate D line\n\ + add %0, %0, #16 @ 1\n\ + ldmia %1!, {r3, r4, ip, lr} @ 4\n\ + stmia %0, {r3, r4, ip, lr} @ 4\n\ + mcr p15, 0, %0, c7, c14, 1 @ 1 clean and invalidate D line\n\ + add %0, %0, #16 @ 1\n\ + subs %2, %2, #1 @ 1\n\ bne 1b @ 1\n\ - mcr p15, 0, r2, c7, c10, 4 @ 1 drain WB\n\ - ldmfd sp!, {r4, pc} @ 3" - : - : "I" (PAGE_SIZE / 32)); + mcr p15, 0, %2, c7, c10, 4 @ 1 drain WB" + : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) + : "2" (PAGE_SIZE / 32) + : "r3", "r4", "ip", "lr"); } void fa_copy_user_highpage(struct page *to, struct page *from, diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c index 49ee0c1a7209..cc819732d9b8 100644 --- a/arch/arm/mm/copypage-feroceon.c +++ b/arch/arm/mm/copypage-feroceon.c @@ -13,58 +13,56 @@ #include <linux/init.h> #include <linux/highmem.h> -static void __naked -feroceon_copy_user_page(void *kto, const void *kfrom) +static void feroceon_copy_user_page(void *kto, const void *kfrom) { - asm("\ - stmfd sp!, {r4-r9, lr} \n\ - mov ip, %2 \n\ -1: mov lr, r1 \n\ - ldmia r1!, {r2 - r9} \n\ - pld [lr, #32] \n\ - pld [lr, #64] \n\ - pld [lr, #96] \n\ - pld [lr, #128] \n\ - pld [lr, #160] \n\ - pld [lr, #192] \n\ - pld [lr, #224] \n\ - stmia r0, {r2 - r9} \n\ - ldmia r1!, {r2 - r9} \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ - stmia r0, {r2 - r9} \n\ - ldmia r1!, {r2 - r9} \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ - stmia r0, {r2 - r9} \n\ - ldmia r1!, {r2 - r9} \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ - stmia r0, {r2 - r9} \n\ - ldmia r1!, {r2 - r9} \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ - stmia r0, {r2 - r9} \n\ - ldmia r1!, {r2 - r9} \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ - stmia r0, {r2 - r9} \n\ - ldmia r1!, {r2 - r9} \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ - stmia r0, {r2 - r9} \n\ - ldmia r1!, {r2 - r9} \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ - stmia r0, {r2 - r9} \n\ - subs ip, ip, #(32 * 8) \n\ - mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\ - add r0, r0, #32 \n\ + int tmp; + + asm volatile ("\ +1: ldmia %1!, {r2 - r7, ip, lr} \n\ + pld [%1, #0] \n\ + pld [%1, #32] \n\ + pld [%1, #64] \n\ + pld [%1, #96] \n\ + pld [%1, #128] \n\ + pld [%1, #160] \n\ + pld [%1, #192] \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + ldmia %1!, {r2 - r7, ip, lr} \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + ldmia %1!, {r2 - r7, ip, lr} \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + ldmia %1!, {r2 - r7, ip, lr} \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + ldmia %1!, {r2 - r7, ip, lr} \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + ldmia %1!, {r2 - r7, ip, lr} \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + ldmia %1!, {r2 - r7, ip, lr} \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + ldmia %1!, {r2 - r7, ip, lr} \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ + stmia %0, {r2 - r7, ip, lr} \n\ + subs %2, %2, #(32 * 8) \n\ + mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ + add %0, %0, #32 \n\ bne 1b \n\ - mcr p15, 0, ip, c7, c10, 4 @ drain WB\n\ - ldmfd sp!, {r4-r9, pc}" - : - : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE)); + mcr p15, 0, %2, c7, c10, 4 @ drain WB" + : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) + : "2" (PAGE_SIZE) + : "r2", "r3", "r4", "r5", "r6", "r7", "ip", "lr"); } void feroceon_copy_user_highpage(struct page *to, struct page *from, diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index 1267e64133b9..b03202cddddb 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -40,12 +40,11 @@ static DEFINE_RAW_SPINLOCK(minicache_lock); * instruction. If your processor does not supply this, you have to write your * own copy_user_highpage that does the right thing. */ -static void __naked -mc_copy_user_page(void *from, void *to) +static void mc_copy_user_page(void *from, void *to) { - asm volatile( - "stmfd sp!, {r4, lr} @ 2\n\ - mov r4, %2 @ 1\n\ + int tmp; + + asm volatile ("\ ldmia %0!, {r2, r3, ip, lr} @ 4\n\ 1: mcr p15, 0, %1, c7, c6, 1 @ 1 invalidate D line\n\ stmia %1!, {r2, r3, ip, lr} @ 4\n\ @@ -55,13 +54,13 @@ mc_copy_user_page(void *from, void *to) mcr p15, 0, %1, c7, c6, 1 @ 1 invalidate D line\n\ stmia %1!, {r2, r3, ip, lr} @ 4\n\ ldmia %0!, {r2, r3, ip, lr} @ 4\n\ - subs r4, r4, #1 @ 1\n\ + subs %2, %2, #1 @ 1\n\ stmia %1!, {r2, r3, ip, lr} @ 4\n\ ldmneia %0!, {r2, r3, ip, lr} @ 4\n\ - bne 1b @ 1\n\ - ldmfd sp!, {r4, pc} @ 3" - : - : "r" (from), "r" (to), "I" (PAGE_SIZE / 64)); + bne 1b @ " + : "+&r" (from), "+&r" (to), "=&r" (tmp) + : "2" (PAGE_SIZE / 64) + : "r2", "r3", "ip", "lr"); } void v4_mc_copy_user_highpage(struct page *to, struct page *from, @@ -70,7 +69,7 @@ void v4_mc_copy_user_highpage(struct page *to, struct page *from, void *kto = kmap_atomic(to); if (!test_and_set_bit(PG_dcache_clean, &from->flags)) - __flush_dcache_page(page_mapping(from), from); + __flush_dcache_page(page_mapping_file(from), from); raw_spin_lock(&minicache_lock); diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c index 067d0fdd630c..cd3e165afeed 100644 --- a/arch/arm/mm/copypage-v4wb.c +++ b/arch/arm/mm/copypage-v4wb.c @@ -22,29 +22,28 @@ * instruction. If your processor does not supply this, you have to write your * own copy_user_highpage that does the right thing. */ -static void __naked -v4wb_copy_user_page(void *kto, const void *kfrom) +static void v4wb_copy_user_page(void *kto, const void *kfrom) { - asm("\ - stmfd sp!, {r4, lr} @ 2\n\ - mov r2, %2 @ 1\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4\n\ -1: mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\ - stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4+1\n\ - stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4\n\ - mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\ - stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4\n\ - subs r2, r2, #1 @ 1\n\ - stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmneia r1!, {r3, r4, ip, lr} @ 4\n\ + int tmp; + + asm volatile ("\ + ldmia %1!, {r3, r4, ip, lr} @ 4\n\ +1: mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\ + stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmia %1!, {r3, r4, ip, lr} @ 4+1\n\ + stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmia %1!, {r3, r4, ip, lr} @ 4\n\ + mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\ + stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmia %1!, {r3, r4, ip, lr} @ 4\n\ + subs %2, %2, #1 @ 1\n\ + stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmneia %1!, {r3, r4, ip, lr} @ 4\n\ bne 1b @ 1\n\ - mcr p15, 0, r1, c7, c10, 4 @ 1 drain WB\n\ - ldmfd sp!, {r4, pc} @ 3" - : - : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE / 64)); + mcr p15, 0, %1, c7, c10, 4 @ 1 drain WB" + : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) + : "2" (PAGE_SIZE / 64) + : "r3", "r4", "ip", "lr"); } void v4wb_copy_user_highpage(struct page *to, struct page *from, diff --git a/arch/arm/mm/copypage-v4wt.c b/arch/arm/mm/copypage-v4wt.c index b85c5da2e510..8614572e1296 100644 --- a/arch/arm/mm/copypage-v4wt.c +++ b/arch/arm/mm/copypage-v4wt.c @@ -20,27 +20,26 @@ * dirty data in the cache. However, we do have to ensure that * subsequent reads are up to date. */ -static void __naked -v4wt_copy_user_page(void *kto, const void *kfrom) +static void v4wt_copy_user_page(void *kto, const void *kfrom) { - asm("\ - stmfd sp!, {r4, lr} @ 2\n\ - mov r2, %2 @ 1\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4\n\ -1: stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4+1\n\ - stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4\n\ - stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmia r1!, {r3, r4, ip, lr} @ 4\n\ - subs r2, r2, #1 @ 1\n\ - stmia r0!, {r3, r4, ip, lr} @ 4\n\ - ldmneia r1!, {r3, r4, ip, lr} @ 4\n\ + int tmp; + + asm volatile ("\ + ldmia %1!, {r3, r4, ip, lr} @ 4\n\ +1: stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmia %1!, {r3, r4, ip, lr} @ 4+1\n\ + stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmia %1!, {r3, r4, ip, lr} @ 4\n\ + stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmia %1!, {r3, r4, ip, lr} @ 4\n\ + subs %2, %2, #1 @ 1\n\ + stmia %0!, {r3, r4, ip, lr} @ 4\n\ + ldmneia %1!, {r3, r4, ip, lr} @ 4\n\ bne 1b @ 1\n\ - mcr p15, 0, r2, c7, c7, 0 @ flush ID cache\n\ - ldmfd sp!, {r4, pc} @ 3" - : - : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE / 64)); + mcr p15, 0, %2, c7, c7, 0 @ flush ID cache" + : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) + : "2" (PAGE_SIZE / 64) + : "r3", "r4", "ip", "lr"); } void v4wt_copy_user_highpage(struct page *to, struct page *from, diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index 70423345da26..a698e575e321 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -76,7 +76,7 @@ static void v6_copy_user_highpage_aliasing(struct page *to, unsigned long kfrom, kto; if (!test_and_set_bit(PG_dcache_clean, &from->flags)) - __flush_dcache_page(page_mapping(from), from); + __flush_dcache_page(page_mapping_file(from), from); /* FIXME: not highmem safe */ discard_old_kernel_data(page_address(to)); diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c index 03a2042aced5..a08158241ad1 100644 --- a/arch/arm/mm/copypage-xsc3.c +++ b/arch/arm/mm/copypage-xsc3.c @@ -21,53 +21,46 @@ /* * XSC3 optimised copy_user_highpage - * r0 = destination - * r1 = source * * The source page may have some clean entries in the cache already, but we * can safely ignore them - break_cow() will flush them out of the cache * if we eventually end up using our copied page. * */ -static void __naked -xsc3_mc_copy_user_page(void *kto, const void *kfrom) +static void xsc3_mc_copy_user_page(void *kto, const void *kfrom) { - asm("\ - stmfd sp!, {r4, r5, lr} \n\ - mov lr, %2 \n\ - \n\ - pld [r1, #0] \n\ - pld [r1, #32] \n\ -1: pld [r1, #64] \n\ - pld [r1, #96] \n\ + int tmp; + + asm volatile ("\ + pld [%1, #0] \n\ + pld [%1, #32] \n\ +1: pld [%1, #64] \n\ + pld [%1, #96] \n\ \n\ -2: ldrd r2, [r1], #8 \n\ - mov ip, r0 \n\ - ldrd r4, [r1], #8 \n\ - mcr p15, 0, ip, c7, c6, 1 @ invalidate\n\ - strd r2, [r0], #8 \n\ - ldrd r2, [r1], #8 \n\ - strd r4, [r0], #8 \n\ - ldrd r4, [r1], #8 \n\ - strd r2, [r0], #8 \n\ - strd r4, [r0], #8 \n\ - ldrd r2, [r1], #8 \n\ - mov ip, r0 \n\ - ldrd r4, [r1], #8 \n\ - mcr p15, 0, ip, c7, c6, 1 @ invalidate\n\ - strd r2, [r0], #8 \n\ - ldrd r2, [r1], #8 \n\ - subs lr, lr, #1 \n\ - strd r4, [r0], #8 \n\ - ldrd r4, [r1], #8 \n\ - strd r2, [r0], #8 \n\ - strd r4, [r0], #8 \n\ +2: ldrd r2, r3, [%1], #8 \n\ + ldrd r4, r5, [%1], #8 \n\ + mcr p15, 0, %0, c7, c6, 1 @ invalidate\n\ + strd r2, r3, [%0], #8 \n\ + ldrd r2, r3, [%1], #8 \n\ + strd r4, r5, [%0], #8 \n\ + ldrd r4, r5, [%1], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r4, r5, [%0], #8 \n\ + ldrd r2, r3, [%1], #8 \n\ + ldrd r4, r5, [%1], #8 \n\ + mcr p15, 0, %0, c7, c6, 1 @ invalidate\n\ + strd r2, r3, [%0], #8 \n\ + ldrd r2, r3, [%1], #8 \n\ + subs %2, %2, #1 \n\ + strd r4, r5, [%0], #8 \n\ + ldrd r4, r5, [%1], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r4, r5, [%0], #8 \n\ bgt 1b \n\ - beq 2b \n\ - \n\ - ldmfd sp!, {r4, r5, pc}" - : - : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE / 64 - 1)); + beq 2b " + : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) + : "2" (PAGE_SIZE / 64 - 1) + : "r2", "r3", "r4", "r5"); } void xsc3_mc_copy_user_highpage(struct page *to, struct page *from, @@ -85,8 +78,6 @@ void xsc3_mc_copy_user_highpage(struct page *to, struct page *from, /* * XScale optimised clear_user_page - * r0 = destination - * r1 = virtual user address of ultimate destination page */ void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr) { @@ -96,10 +87,10 @@ void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr) mov r2, #0 \n\ mov r3, #0 \n\ 1: mcr p15, 0, %0, c7, c6, 1 @ invalidate line\n\ - strd r2, [%0], #8 \n\ - strd r2, [%0], #8 \n\ - strd r2, [%0], #8 \n\ - strd r2, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ subs r1, r1, #1 \n\ bne 1b" : "=r" (ptr) diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c index 0fb85025344d..63b921936754 100644 --- a/arch/arm/mm/copypage-xscale.c +++ b/arch/arm/mm/copypage-xscale.c @@ -36,52 +36,51 @@ static DEFINE_RAW_SPINLOCK(minicache_lock); * Dcache aliasing issue. The writes will be forwarded to the write buffer, * and merged as appropriate. */ -static void __naked -mc_copy_user_page(void *from, void *to) +static void mc_copy_user_page(void *from, void *to) { + int tmp; + /* * Strangely enough, best performance is achieved * when prefetching destination as well. (NP) */ - asm volatile( - "stmfd sp!, {r4, r5, lr} \n\ - mov lr, %2 \n\ - pld [r0, #0] \n\ - pld [r0, #32] \n\ - pld [r1, #0] \n\ - pld [r1, #32] \n\ -1: pld [r0, #64] \n\ - pld [r0, #96] \n\ - pld [r1, #64] \n\ - pld [r1, #96] \n\ -2: ldrd r2, [r0], #8 \n\ - ldrd r4, [r0], #8 \n\ - mov ip, r1 \n\ - strd r2, [r1], #8 \n\ - ldrd r2, [r0], #8 \n\ - strd r4, [r1], #8 \n\ - ldrd r4, [r0], #8 \n\ - strd r2, [r1], #8 \n\ - strd r4, [r1], #8 \n\ + asm volatile ("\ + pld [%0, #0] \n\ + pld [%0, #32] \n\ + pld [%1, #0] \n\ + pld [%1, #32] \n\ +1: pld [%0, #64] \n\ + pld [%0, #96] \n\ + pld [%1, #64] \n\ + pld [%1, #96] \n\ +2: ldrd r2, r3, [%0], #8 \n\ + ldrd r4, r5, [%0], #8 \n\ + mov ip, %1 \n\ + strd r2, r3, [%1], #8 \n\ + ldrd r2, r3, [%0], #8 \n\ + strd r4, r5, [%1], #8 \n\ + ldrd r4, r5, [%0], #8 \n\ + strd r2, r3, [%1], #8 \n\ + strd r4, r5, [%1], #8 \n\ mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ - ldrd r2, [r0], #8 \n\ + ldrd r2, r3, [%0], #8 \n\ mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ - ldrd r4, [r0], #8 \n\ - mov ip, r1 \n\ - strd r2, [r1], #8 \n\ - ldrd r2, [r0], #8 \n\ - strd r4, [r1], #8 \n\ - ldrd r4, [r0], #8 \n\ - strd r2, [r1], #8 \n\ - strd r4, [r1], #8 \n\ + ldrd r4, r5, [%0], #8 \n\ + mov ip, %1 \n\ + strd r2, r3, [%1], #8 \n\ + ldrd r2, r3, [%0], #8 \n\ + strd r4, r5, [%1], #8 \n\ + ldrd r4, r5, [%0], #8 \n\ + strd r2, r3, [%1], #8 \n\ + strd r4, r5, [%1], #8 \n\ mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ - subs lr, lr, #1 \n\ + subs %2, %2, #1 \n\ mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ bgt 1b \n\ - beq 2b \n\ - ldmfd sp!, {r4, r5, pc} " - : - : "r" (from), "r" (to), "I" (PAGE_SIZE / 64 - 1)); + beq 2b " + : "+&r" (from), "+&r" (to), "=&r" (tmp) + : "2" (PAGE_SIZE / 64 - 1) + : "r2", "r3", "r4", "r5", "ip"); } void xscale_mc_copy_user_highpage(struct page *to, struct page *from, @@ -90,7 +89,7 @@ void xscale_mc_copy_user_highpage(struct page *to, struct page *from, void *kto = kmap_atomic(to); if (!test_and_set_bit(PG_dcache_clean, &from->flags)) - __flush_dcache_page(page_mapping(from), from); + __flush_dcache_page(page_mapping_file(from), from); raw_spin_lock(&minicache_lock); @@ -115,10 +114,10 @@ xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr) mov r2, #0 \n\ mov r3, #0 \n\ 1: mov ip, %0 \n\ - strd r2, [%0], #8 \n\ - strd r2, [%0], #8 \n\ - strd r2, [%0], #8 \n\ - strd r2, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ + strd r2, r3, [%0], #8 \n\ mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ subs r1, r1, #1 \n\ mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index 619f24a42d09..712416ecd8e6 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -47,7 +47,8 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, */ if (attrs & DMA_ATTR_NON_CONSISTENT) - return dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + return dma_direct_alloc_pages(dev, size, dma_handle, gfp, + attrs); ret = dma_alloc_from_global_coherent(size, dma_handle); @@ -70,7 +71,7 @@ static void arm_nommu_dma_free(struct device *dev, size_t size, unsigned long attrs) { if (attrs & DMA_ATTR_NON_CONSISTENT) { - dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); + dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); } else { int ret = dma_release_from_global_coherent(get_order(size), cpu_addr); @@ -90,7 +91,7 @@ static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) return ret; - return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); + return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); } @@ -237,16 +238,3 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, set_dma_ops(dev, dma_ops); } - -void arch_teardown_dma_ops(struct device *dev) -{ -} - -#define PREALLOC_DMA_DEBUG_ENTRIES 4096 - -static int __init dma_debug_do_init(void) -{ - dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); - return 0; -} -core_initcall(dma_debug_do_init); diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index ada8eb206a90..661fe48ab78d 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -9,7 +9,6 @@ * * DMA uncached mapping support. */ -#include <linux/bootmem.h> #include <linux/module.h> #include <linux/mm.h> #include <linux/genalloc.h> @@ -594,7 +593,7 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size, struct page *page; void *ptr = NULL; - page = dma_alloc_from_contiguous(dev, count, order, gfp); + page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); if (!page) return NULL; @@ -831,7 +830,7 @@ static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, unsigned long attrs) { int ret; - unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + unsigned long nr_vma_pages = vma_pages(vma); unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long pfn = dma_to_pfn(dev, dma_addr); unsigned long off = vma->vm_pgoff; @@ -1151,14 +1150,10 @@ int arm_dma_supported(struct device *dev, u64 mask) return __dma_supported(dev, mask, false); } -#define PREALLOC_DMA_DEBUG_ENTRIES 4096 - -static int __init dma_debug_do_init(void) +static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent) { - dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); - return 0; + return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; } -core_initcall(dma_debug_do_init); #ifdef CONFIG_ARM_DMA_USE_IOMMU @@ -1303,7 +1298,8 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, unsigned long order = get_order(size); struct page *page; - page = dma_alloc_from_contiguous(dev, count, order, gfp); + page = dma_alloc_from_contiguous(dev, count, order, + gfp & __GFP_NOWARN); if (!page) goto error; @@ -2171,8 +2167,8 @@ arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size) goto err; mapping->bitmap_size = bitmap_size; - mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *), - GFP_KERNEL); + mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *), + GFP_KERNEL); if (!mapping->bitmaps) goto err2; @@ -2305,7 +2301,7 @@ void arm_iommu_detach_device(struct device *dev) iommu_detach_device(mapping->domain, dev); kref_put(&mapping->kref, release_iommu_mapping); to_dma_iommu_mapping(dev) = NULL; - set_dma_ops(dev, NULL); + set_dma_ops(dev, arm_get_dma_map_ops(dev->archdata.dma_coherent)); pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); } @@ -2366,11 +2362,6 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) { } #endif /* CONFIG_ARM_DMA_USE_IOMMU */ -static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent) -{ - return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; -} - void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, const struct iommu_ops *iommu, bool coherent) { diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index d9e0d00a6699..4d75dae5ac96 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -195,7 +195,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, if (page == ZERO_PAGE(0)) return; - mapping = page_mapping(page); + mapping = page_mapping_file(page); if (!test_and_set_bit(PG_dcache_clean, &page->flags)) __flush_dcache_page(mapping, page); if (mapping) { diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 3b1ba003c4f9..58f69fa07df9 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -161,8 +161,6 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr, unsigned int fsr, unsigned int sig, int code, struct pt_regs *regs) { - struct siginfo si; - if (addr > TASK_SIZE) harden_branch_predictor(); @@ -175,15 +173,17 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr, show_regs(regs); } #endif +#ifndef CONFIG_KUSER_HELPERS + if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000)) + printk_ratelimited(KERN_DEBUG + "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n", + tsk->comm, addr); +#endif tsk->thread.address = addr; tsk->thread.error_code = fsr; tsk->thread.trap_no = 14; - si.si_signo = sig; - si.si_errno = 0; - si.si_code = code; - si.si_addr = (void __user *)addr; - force_sig_info(sig, &si, tsk); + force_sig_fault(sig, code, (void __user *)addr, tsk); } void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs) @@ -222,12 +222,12 @@ static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma) return vma->vm_flags & mask ? false : true; } -static int __kprobes +static vm_fault_t __kprobes __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, unsigned int flags, struct task_struct *tsk) { struct vm_area_struct *vma; - int fault; + vm_fault_t fault; vma = find_vma(mm, addr); fault = VM_FAULT_BADMAP; @@ -262,7 +262,8 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { struct task_struct *tsk; struct mm_struct *mm; - int fault, sig, code; + int sig, code; + vm_fault_t fault; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; if (notify_page_fault(regs, fsr)) @@ -551,7 +552,6 @@ asmlinkage void do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { const struct fsr_info *inf = fsr_info + fsr_fs(fsr); - struct siginfo info; if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs)) return; @@ -560,11 +560,8 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) inf->name, fsr, addr); show_pte(current->mm, addr); - info.si_signo = inf->sig; - info.si_errno = 0; - info.si_code = inf->code; - info.si_addr = (void __user *)addr; - arm_notify_die("", regs, &info, fsr, 0); + arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, + fsr, 0); } void __init @@ -584,7 +581,6 @@ asmlinkage void do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) { const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr); - struct siginfo info; if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) return; @@ -592,11 +588,8 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", inf->name, ifsr, addr); - info.si_signo = inf->sig; - info.si_errno = 0; - info.si_code = inf->code; - info.si_addr = (void __user *)addr; - arm_notify_die("", regs, &info, ifsr, 0); + arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, + ifsr, 0); } /* diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index f1e6190aa7ea..58469623b015 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -285,7 +285,7 @@ void __sync_icache_dcache(pte_t pteval) page = pfn_to_page(pfn); if (cache_is_vipt_aliasing()) - mapping = page_mapping(page); + mapping = page_mapping_file(page); else mapping = NULL; @@ -333,7 +333,7 @@ void flush_dcache_page(struct page *page) return; } - mapping = page_mapping(page); + mapping = page_mapping_file(page); if (!cache_ops_need_broadcast() && mapping && !page_mapcount(page)) @@ -363,7 +363,7 @@ void flush_kernel_dcache_page(struct page *page) if (cache_is_vivt() || cache_is_vipt_aliasing()) { struct address_space *mapping; - mapping = page_mapping(page); + mapping = page_mapping_file(page); if (!mapping || mapping_mapped(mapping)) { void *addr; diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index bd6f4513539a..32e4845af2b6 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -11,7 +11,6 @@ #include <linux/errno.h> #include <linux/swap.h> #include <linux/init.h> -#include <linux/bootmem.h> #include <linux/mman.h> #include <linux/sched/signal.h> #include <linux/sched/task.h> @@ -508,7 +507,7 @@ void __init mem_init(void) /* this will put all unused low memory onto the freelists */ free_unused_memmap(); - free_all_bootmem(); + memblock_free_all(); #ifdef CONFIG_SA1111 /* now that our DMA memory is actually so designated, we can free it */ @@ -736,20 +735,29 @@ static int __mark_rodata_ro(void *unused) return 0; } +static int kernel_set_to_readonly __read_mostly; + void mark_rodata_ro(void) { + kernel_set_to_readonly = 1; stop_machine(__mark_rodata_ro, NULL, NULL); debug_checkwx(); } void set_kernel_text_rw(void) { + if (!kernel_set_to_readonly) + return; + set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false, current->active_mm); } void set_kernel_text_ro(void) { + if (!kernel_set_to_readonly) + return; + set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, current->active_mm); } @@ -758,20 +766,9 @@ void set_kernel_text_ro(void) static inline void fix_kernmem_perms(void) { } #endif /* CONFIG_STRICT_KERNEL_RWX */ -void free_tcmmem(void) -{ -#ifdef CONFIG_HAVE_TCM - extern char __tcm_start, __tcm_end; - - poison_init_mem(&__tcm_start, &__tcm_end - &__tcm_start); - free_reserved_area(&__tcm_start, &__tcm_end, -1, "TCM link"); -#endif -} - void free_initmem(void) { fix_kernmem_perms(); - free_tcmmem(); poison_init_mem(__init_begin, __init_end - __init_begin); if (!machine_is_integrator() && !machine_is_cintegrator()) diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index fc91205ff46c..5bf9443cfbaa 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -473,7 +473,7 @@ void pci_ioremap_set_mem_type(int mem_type) int pci_ioremap_io(unsigned int offset, phys_addr_t phys_addr) { - BUG_ON(offset + SZ_64K > IO_SPACE_LIMIT); + BUG_ON(offset + SZ_64K - 1 > IO_SPACE_LIMIT); return ioremap_page_range(PCI_IO_VIRT_BASE + offset, PCI_IO_VIRT_BASE + offset + SZ_64K, diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index eb1de66517d5..f866870db749 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -21,20 +21,20 @@ #define MIN_GAP (128*1024*1024UL) #define MAX_GAP ((TASK_SIZE)/6*5) -static int mmap_is_legacy(void) +static int mmap_is_legacy(struct rlimit *rlim_stack) { if (current->personality & ADDR_COMPAT_LAYOUT) return 1; - if (rlimit(RLIMIT_STACK) == RLIM_INFINITY) + if (rlim_stack->rlim_cur == RLIM_INFINITY) return 1; return sysctl_legacy_va_layout; } -static unsigned long mmap_base(unsigned long rnd) +static unsigned long mmap_base(unsigned long rnd, struct rlimit *rlim_stack) { - unsigned long gap = rlimit(RLIMIT_STACK); + unsigned long gap = rlim_stack->rlim_cur; if (gap < MIN_GAP) gap = MIN_GAP; @@ -180,18 +180,18 @@ unsigned long arch_mmap_rnd(void) return rnd << PAGE_SHIFT; } -void arch_pick_mmap_layout(struct mm_struct *mm) +void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) { unsigned long random_factor = 0UL; if (current->flags & PF_RANDOMIZE) random_factor = arch_mmap_rnd(); - if (mmap_is_legacy()) { + if (mmap_is_legacy(rlim_stack)) { mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; mm->get_unmapped_area = arch_get_unmapped_area; } else { - mm->mmap_base = mmap_base(random_factor); + mm->mmap_base = mmap_base(random_factor, rlim_stack); mm->get_unmapped_area = arch_get_unmapped_area_topdown; } } diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index e46a6a446cdd..f5cc1ccfea3d 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -721,7 +721,7 @@ EXPORT_SYMBOL(phys_mem_access_prot); static void __init *early_alloc_aligned(unsigned long sz, unsigned long align) { - void *ptr = __va(memblock_alloc(sz, align)); + void *ptr = __va(memblock_phys_alloc(sz, align)); memset(ptr, 0, sz); return ptr; } diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 7c087961b7ce..7d67c70bbded 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -53,7 +53,8 @@ static inline bool security_extensions_enabled(void) { /* Check CPUID Identification Scheme before ID_PFR1 read */ if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) - return !!cpuid_feature_extract(CPUID_EXT_PFR1, 4); + return cpuid_feature_extract(CPUID_EXT_PFR1, 4) || + cpuid_feature_extract(CPUID_EXT_PFR1, 20); return 0; } @@ -99,6 +100,38 @@ void __init arm_mm_memblock_reserve(void) memblock_reserve(0, 1); } +static void __init adjust_lowmem_bounds_mpu(void) +{ + unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA; + + switch (pmsa) { + case MMFR0_PMSAv7: + pmsav7_adjust_lowmem_bounds(); + break; + case MMFR0_PMSAv8: + pmsav8_adjust_lowmem_bounds(); + break; + default: + break; + } +} + +static void __init mpu_setup(void) +{ + unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA; + + switch (pmsa) { + case MMFR0_PMSAv7: + pmsav7_setup(); + break; + case MMFR0_PMSAv8: + pmsav8_setup(); + break; + default: + break; + } +} + void __init adjust_lowmem_bounds(void) { phys_addr_t end; diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index 61e281cb29fb..a1606d950251 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -20,7 +20,7 @@ #include "mm.h" #ifdef CONFIG_ARM_LPAE -#define __pgd_alloc() kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL) +#define __pgd_alloc() kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL) #define __pgd_free(pgd) kfree(pgd) #else #define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2) diff --git a/arch/arm/mm/pmsa-v7.c b/arch/arm/mm/pmsa-v7.c index e2853bfff74e..699fa2e88725 100644 --- a/arch/arm/mm/pmsa-v7.c +++ b/arch/arm/mm/pmsa-v7.c @@ -102,7 +102,7 @@ static inline u32 irbar_read(void) static inline void rgnr_write(u32 v) { - writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RNR); + writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv7_RNR); } /* Data-side / unified region attributes */ @@ -110,28 +110,28 @@ static inline void rgnr_write(u32 v) /* Region access control register */ static inline void dracr_write(u32 v) { - u32 rsr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(15, 0); + u32 rsr = readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RASR) & GENMASK(15, 0); - writel_relaxed((v << 16) | rsr, BASEADDR_V7M_SCB + MPU_RASR); + writel_relaxed((v << 16) | rsr, BASEADDR_V7M_SCB + PMSAv7_RASR); } /* Region size register */ static inline void drsr_write(u32 v) { - u32 racr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(31, 16); + u32 racr = readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RASR) & GENMASK(31, 16); - writel_relaxed(v | racr, BASEADDR_V7M_SCB + MPU_RASR); + writel_relaxed(v | racr, BASEADDR_V7M_SCB + PMSAv7_RASR); } /* Region base address register */ static inline void drbar_write(u32 v) { - writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RBAR); + writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv7_RBAR); } static inline u32 drbar_read(void) { - return readl_relaxed(BASEADDR_V7M_SCB + MPU_RBAR); + return readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RBAR); } /* ARMv7-M only supports a unified MPU, so I-side operations are nop */ @@ -143,11 +143,6 @@ static inline unsigned long irbar_read(void) {return 0;} #endif -static int __init mpu_present(void) -{ - return ((read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA) == MMFR0_PMSAv7); -} - static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct region *region) { unsigned long subreg, bslots, sslots; @@ -161,7 +156,7 @@ static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct r bdiff = base - abase; sdiff = p2size - asize; - subreg = p2size / MPU_NR_SUBREGS; + subreg = p2size / PMSAv7_NR_SUBREGS; if ((bdiff % subreg) || (sdiff % subreg)) return false; @@ -172,17 +167,17 @@ static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct r if (bslots || sslots) { int i; - if (subreg < MPU_MIN_SUBREG_SIZE) + if (subreg < PMSAv7_MIN_SUBREG_SIZE) return false; - if (bslots + sslots > MPU_NR_SUBREGS) + if (bslots + sslots > PMSAv7_NR_SUBREGS) return false; for (i = 0; i < bslots; i++) _set_bit(i, ®ion->subreg); for (i = 1; i <= sslots; i++) - _set_bit(MPU_NR_SUBREGS - i, ®ion->subreg); + _set_bit(PMSAv7_NR_SUBREGS - i, ®ion->subreg); } region->base = abase; @@ -233,7 +228,7 @@ static int __init allocate_region(phys_addr_t base, phys_addr_t size, } /* MPU initialisation functions */ -void __init adjust_lowmem_bounds_mpu(void) +void __init pmsav7_adjust_lowmem_bounds(void) { phys_addr_t specified_mem_size = 0, total_mem_size = 0; struct memblock_region *reg; @@ -243,10 +238,7 @@ void __init adjust_lowmem_bounds_mpu(void) unsigned int mem_max_regions; int num, i; - if (!mpu_present()) - return; - - /* Free-up MPU_PROBE_REGION */ + /* Free-up PMSAv7_PROBE_REGION */ mpu_min_region_order = __mpu_min_region_order(); /* How many regions are supported */ @@ -301,12 +293,12 @@ void __init adjust_lowmem_bounds_mpu(void) num = allocate_region(mem_start, specified_mem_size, mem_max_regions, mem); for (i = 0; i < num; i++) { - unsigned long subreg = mem[i].size / MPU_NR_SUBREGS; + unsigned long subreg = mem[i].size / PMSAv7_NR_SUBREGS; total_mem_size += mem[i].size - subreg * hweight_long(mem[i].subreg); pr_debug("MPU: base %pa size %pa disable subregions: %*pbl\n", - &mem[i].base, &mem[i].size, MPU_NR_SUBREGS, &mem[i].subreg); + &mem[i].base, &mem[i].size, PMSAv7_NR_SUBREGS, &mem[i].subreg); } if (total_mem_size != specified_mem_size) { @@ -349,7 +341,7 @@ static int __init __mpu_min_region_order(void) u32 drbar_result, irbar_result; /* We've kept a region free for this probing */ - rgnr_write(MPU_PROBE_REGION); + rgnr_write(PMSAv7_PROBE_REGION); isb(); /* * As per ARM ARM, write 0xFFFFFFFC to DRBAR to find the minimum @@ -388,8 +380,8 @@ static int __init mpu_setup_region(unsigned int number, phys_addr_t start, return -ENOMEM; /* Writing N to bits 5:1 (RSR_SZ) specifies region size 2^N+1 */ - size_data = ((size_order - 1) << MPU_RSR_SZ) | 1 << MPU_RSR_EN; - size_data |= subregions << MPU_RSR_SD; + size_data = ((size_order - 1) << PMSAv7_RSR_SZ) | 1 << PMSAv7_RSR_EN; + size_data |= subregions << PMSAv7_RSR_SD; if (need_flush) flush_cache_all(); @@ -424,18 +416,15 @@ static int __init mpu_setup_region(unsigned int number, phys_addr_t start, /* * Set up default MPU regions, doing nothing if there is no MPU */ -void __init mpu_setup(void) +void __init pmsav7_setup(void) { int i, region = 0, err = 0; - if (!mpu_present()) - return; - /* Setup MPU (order is important) */ /* Background */ err |= mpu_setup_region(region++, 0, 32, - MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0RW, + PMSAv7_ACR_XN | PMSAv7_RGN_STRONGLY_ORDERED | PMSAv7_AP_PL1RW_PL0RW, 0, false); #ifdef CONFIG_XIP_KERNEL @@ -448,13 +437,13 @@ void __init mpu_setup(void) * with BG region (which is uncachable), thus we need * to clean and invalidate cache. */ - bool need_flush = region == MPU_RAM_REGION; + bool need_flush = region == PMSAv7_RAM_REGION; if (!xip[i].size) continue; err |= mpu_setup_region(region++, xip[i].base, ilog2(xip[i].size), - MPU_AP_PL1RO_PL0NA | MPU_RGN_NORMAL, + PMSAv7_AP_PL1RO_PL0NA | PMSAv7_RGN_NORMAL, xip[i].subreg, need_flush); } #endif @@ -465,14 +454,14 @@ void __init mpu_setup(void) continue; err |= mpu_setup_region(region++, mem[i].base, ilog2(mem[i].size), - MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL, + PMSAv7_AP_PL1RW_PL0RW | PMSAv7_RGN_NORMAL, mem[i].subreg, false); } /* Vectors */ #ifndef CONFIG_CPU_V7M err |= mpu_setup_region(region++, vectors_base, ilog2(2 * PAGE_SIZE), - MPU_AP_PL1RW_PL0NA | MPU_RGN_NORMAL, + PMSAv7_AP_PL1RW_PL0NA | PMSAv7_RGN_NORMAL, 0, false); #endif if (err) { diff --git a/arch/arm/mm/pmsa-v8.c b/arch/arm/mm/pmsa-v8.c new file mode 100644 index 000000000000..617a83def88a --- /dev/null +++ b/arch/arm/mm/pmsa-v8.c @@ -0,0 +1,307 @@ +/* + * Based on linux/arch/arm/pmsa-v7.c + * + * ARM PMSAv8 supporting functions. + */ + +#include <linux/memblock.h> +#include <linux/range.h> + +#include <asm/cp15.h> +#include <asm/cputype.h> +#include <asm/mpu.h> + +#include <asm/memory.h> +#include <asm/sections.h> + +#include "mm.h" + +#ifndef CONFIG_CPU_V7M + +#define PRSEL __ACCESS_CP15(c6, 0, c2, 1) +#define PRBAR __ACCESS_CP15(c6, 0, c3, 0) +#define PRLAR __ACCESS_CP15(c6, 0, c3, 1) + +static inline u32 prlar_read(void) +{ + return read_sysreg(PRLAR); +} + +static inline u32 prbar_read(void) +{ + return read_sysreg(PRBAR); +} + +static inline void prsel_write(u32 v) +{ + write_sysreg(v, PRSEL); +} + +static inline void prbar_write(u32 v) +{ + write_sysreg(v, PRBAR); +} + +static inline void prlar_write(u32 v) +{ + write_sysreg(v, PRLAR); +} +#else + +static inline u32 prlar_read(void) +{ + return readl_relaxed(BASEADDR_V7M_SCB + PMSAv8_RLAR); +} + +static inline u32 prbar_read(void) +{ + return readl_relaxed(BASEADDR_V7M_SCB + PMSAv8_RBAR); +} + +static inline void prsel_write(u32 v) +{ + writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RNR); +} + +static inline void prbar_write(u32 v) +{ + writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RBAR); +} + +static inline void prlar_write(u32 v) +{ + writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RLAR); +} + +#endif + +static struct range __initdata io[MPU_MAX_REGIONS]; +static struct range __initdata mem[MPU_MAX_REGIONS]; + +static unsigned int __initdata mpu_max_regions; + +static __init bool is_region_fixed(int number) +{ + switch (number) { + case PMSAv8_XIP_REGION: + case PMSAv8_KERNEL_REGION: + return true; + default: + return false; + } +} + +void __init pmsav8_adjust_lowmem_bounds(void) +{ + phys_addr_t mem_end; + struct memblock_region *reg; + bool first = true; + + for_each_memblock(memory, reg) { + if (first) { + phys_addr_t phys_offset = PHYS_OFFSET; + + /* + * Initially only use memory continuous from + * PHYS_OFFSET */ + if (reg->base != phys_offset) + panic("First memory bank must be contiguous from PHYS_OFFSET"); + mem_end = reg->base + reg->size; + first = false; + } else { + /* + * memblock auto merges contiguous blocks, remove + * all blocks afterwards in one go (we can't remove + * blocks separately while iterating) + */ + pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n", + &mem_end, ®->base); + memblock_remove(reg->base, 0 - reg->base); + break; + } + } +} + +static int __init __mpu_max_regions(void) +{ + static int max_regions; + u32 mpuir; + + if (max_regions) + return max_regions; + + mpuir = read_cpuid_mputype(); + + max_regions = (mpuir & MPUIR_DREGION_SZMASK) >> MPUIR_DREGION; + + return max_regions; +} + +static int __init __pmsav8_setup_region(unsigned int number, u32 bar, u32 lar) +{ + if (number > mpu_max_regions + || number >= MPU_MAX_REGIONS) + return -ENOENT; + + dsb(); + prsel_write(number); + isb(); + prbar_write(bar); + prlar_write(lar); + + mpu_rgn_info.rgns[number].prbar = bar; + mpu_rgn_info.rgns[number].prlar = lar; + + mpu_rgn_info.used++; + + return 0; +} + +static int __init pmsav8_setup_ram(unsigned int number, phys_addr_t start,phys_addr_t end) +{ + u32 bar, lar; + + if (is_region_fixed(number)) + return -EINVAL; + + bar = start; + lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);; + + bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED; + lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN; + + return __pmsav8_setup_region(number, bar, lar); +} + +static int __init pmsav8_setup_io(unsigned int number, phys_addr_t start,phys_addr_t end) +{ + u32 bar, lar; + + if (is_region_fixed(number)) + return -EINVAL; + + bar = start; + lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);; + + bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN; + lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN; + + return __pmsav8_setup_region(number, bar, lar); +} + +static int __init pmsav8_setup_fixed(unsigned int number, phys_addr_t start,phys_addr_t end) +{ + u32 bar, lar; + + if (!is_region_fixed(number)) + return -EINVAL; + + bar = start; + lar = (end - 1) & ~(PMSAv8_MINALIGN - 1); + + bar |= PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED; + lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN; + + prsel_write(number); + isb(); + + if (prbar_read() != bar || prlar_read() != lar) + return -EINVAL; + + /* Reserved region was set up early, we just need a record for secondaries */ + mpu_rgn_info.rgns[number].prbar = bar; + mpu_rgn_info.rgns[number].prlar = lar; + + mpu_rgn_info.used++; + + return 0; +} + +#ifndef CONFIG_CPU_V7M +static int __init pmsav8_setup_vector(unsigned int number, phys_addr_t start,phys_addr_t end) +{ + u32 bar, lar; + + if (number == PMSAv8_KERNEL_REGION) + return -EINVAL; + + bar = start; + lar = (end - 1) & ~(PMSAv8_MINALIGN - 1); + + bar |= PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED; + lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN; + + return __pmsav8_setup_region(number, bar, lar); +} +#endif + +void __init pmsav8_setup(void) +{ + int i, err = 0; + int region = PMSAv8_KERNEL_REGION; + + /* How many regions are supported ? */ + mpu_max_regions = __mpu_max_regions(); + + /* RAM: single chunk of memory */ + add_range(mem, ARRAY_SIZE(mem), 0, memblock.memory.regions[0].base, + memblock.memory.regions[0].base + memblock.memory.regions[0].size); + + /* IO: cover full 4G range */ + add_range(io, ARRAY_SIZE(io), 0, 0, 0xffffffff); + + /* RAM and IO: exclude kernel */ + subtract_range(mem, ARRAY_SIZE(mem), __pa(KERNEL_START), __pa(KERNEL_END)); + subtract_range(io, ARRAY_SIZE(io), __pa(KERNEL_START), __pa(KERNEL_END)); + +#ifdef CONFIG_XIP_KERNEL + /* RAM and IO: exclude xip */ + subtract_range(mem, ARRAY_SIZE(mem), CONFIG_XIP_PHYS_ADDR, __pa(_exiprom)); + subtract_range(io, ARRAY_SIZE(io), CONFIG_XIP_PHYS_ADDR, __pa(_exiprom)); +#endif + +#ifndef CONFIG_CPU_V7M + /* RAM and IO: exclude vectors */ + subtract_range(mem, ARRAY_SIZE(mem), vectors_base, vectors_base + 2 * PAGE_SIZE); + subtract_range(io, ARRAY_SIZE(io), vectors_base, vectors_base + 2 * PAGE_SIZE); +#endif + /* IO: exclude RAM */ + for (i = 0; i < ARRAY_SIZE(mem); i++) + subtract_range(io, ARRAY_SIZE(io), mem[i].start, mem[i].end); + + /* Now program MPU */ + +#ifdef CONFIG_XIP_KERNEL + /* ROM */ + err |= pmsav8_setup_fixed(PMSAv8_XIP_REGION, CONFIG_XIP_PHYS_ADDR, __pa(_exiprom)); +#endif + /* Kernel */ + err |= pmsav8_setup_fixed(region++, __pa(KERNEL_START), __pa(KERNEL_END)); + + + /* IO */ + for (i = 0; i < ARRAY_SIZE(io); i++) { + if (!io[i].end) + continue; + + err |= pmsav8_setup_io(region++, io[i].start, io[i].end); + } + + /* RAM */ + for (i = 0; i < ARRAY_SIZE(mem); i++) { + if (!mem[i].end) + continue; + + err |= pmsav8_setup_ram(region++, mem[i].start, mem[i].end); + } + + /* Vectors */ +#ifndef CONFIG_CPU_V7M + err |= pmsav8_setup_vector(region++, vectors_base, vectors_base + 2 * PAGE_SIZE); +#endif + if (err) + pr_warn("MPU region initialization failure! %d", err); + else + pr_info("Using ARM PMSAv8 Compliant MPU. Used %d of %d regions\n", + mpu_rgn_info.used, mpu_max_regions); +} diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index fccd4d99f505..339eb17c9808 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -279,6 +279,7 @@ ENDPROC(cpu_pj4b_do_resume) __v7_ca5mp_setup: __v7_ca9mp_setup: __v7_cr7mp_setup: +__v7_cr8mp_setup: mov r10, #(1 << 0) @ Cache/TLB ops broadcasting b 1f __v7_ca7mp_setup: @@ -716,6 +717,16 @@ __v7_cr7mp_proc_info: .size __v7_cr7mp_proc_info, . - __v7_cr7mp_proc_info /* + * ARM Ltd. Cortex R8 processor. + */ + .type __v7_cr8mp_proc_info, #object +__v7_cr8mp_proc_info: + .long 0x410fc180 + .long 0xff0ffff0 + __v7_proc __v7_cr8mp_proc_info, __v7_cr8mp_setup + .size __v7_cr8mp_proc_info, . - __v7_cr8mp_proc_info + + /* * ARM Ltd. Cortex A7 processor. */ .type __v7_ca7mp_proc_info, #object diff --git a/arch/arm/mm/pv-fixup-asm.S b/arch/arm/mm/pv-fixup-asm.S index 1867f3e43016..fd2ff9034d17 100644 --- a/arch/arm/mm/pv-fixup-asm.S +++ b/arch/arm/mm/pv-fixup-asm.S @@ -33,10 +33,10 @@ ENTRY(lpae_pgtables_remap_asm) add r7, r2, #0x1000 add r6, r7, r6, lsr #SECTION_SHIFT - L2_ORDER add r7, r7, #PAGE_OFFSET >> (SECTION_SHIFT - L2_ORDER) -1: ldrd r4, [r7] +1: ldrd r4, r5, [r7] adds r4, r4, r0 adc r5, r5, r1 - strd r4, [r7], #1 << L2_ORDER + strd r4, r5, [r7], #1 << L2_ORDER cmp r7, r6 bls 1b @@ -44,22 +44,22 @@ ENTRY(lpae_pgtables_remap_asm) add r7, r2, #0x1000 add r7, r7, r3, lsr #SECTION_SHIFT - L2_ORDER bic r7, r7, #(1 << L2_ORDER) - 1 - ldrd r4, [r7] + ldrd r4, r5, [r7] adds r4, r4, r0 adc r5, r5, r1 - strd r4, [r7], #1 << L2_ORDER - ldrd r4, [r7] + strd r4, r5, [r7], #1 << L2_ORDER + ldrd r4, r5, [r7] adds r4, r4, r0 adc r5, r5, r1 - strd r4, [r7] + strd r4, r5, [r7] /* Update level 1 entries */ mov r6, #4 mov r7, r2 -2: ldrd r4, [r7] +2: ldrd r4, r5, [r7] adds r4, r4, r0 adc r5, r5, r1 - strd r4, [r7], #1 << L1_ORDER + strd r4, r5, [r7], #1 << L1_ORDER subs r6, r6, #1 bne 2b diff --git a/arch/arm/mm/tcm.h b/arch/arm/mm/tcm.h index 8015ad434a40..24101925fe64 100644 --- a/arch/arm/mm/tcm.h +++ b/arch/arm/mm/tcm.h @@ -11,7 +11,7 @@ void __init tcm_init(void); #else /* No TCM support, just blank inlines to be optimized out */ -inline void tcm_init(void) +static inline void tcm_init(void) { } #endif |
