summaryrefslogtreecommitdiffstats
path: root/arch/arm/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm')
-rw-r--r--arch/arm/mm/Kconfig8
-rw-r--r--arch/arm/mm/Makefile3
-rw-r--r--arch/arm/mm/dma-mapping.c13
-rw-r--r--arch/arm/mm/mmu.c173
-rw-r--r--arch/arm/mm/nommu.c9
-rw-r--r--arch/arm/mm/proc-arm1020.S2
-rw-r--r--arch/arm/mm/proc-arm1020e.S2
-rw-r--r--arch/arm/mm/proc-arm925.S3
-rw-r--r--arch/arm/mm/proc-feroceon.S1
-rw-r--r--arch/arm/mm/proc-v7-2level.S6
-rw-r--r--arch/arm/mm/proc-v7-3level.S14
-rw-r--r--arch/arm/mm/proc-v7.S182
-rw-r--r--arch/arm/mm/pv-fixup-asm.S88
13 files changed, 285 insertions, 219 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 54cc211f866b..7c6b976ab8d3 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -6,7 +6,7 @@ comment "Processor Type"
# ARM7TDMI
config CPU_ARM7TDMI
- bool "Support ARM7TDMI processor"
+ bool
depends on !MMU
select CPU_32v4T
select CPU_ABRT_LV4T
@@ -56,7 +56,7 @@ config CPU_ARM740T
# ARM9TDMI
config CPU_ARM9TDMI
- bool "Support ARM9TDMI processor"
+ bool
depends on !MMU
select CPU_32v4T
select CPU_ABRT_NOMMU
@@ -640,6 +640,10 @@ config ARM_LPAE
If unsure, say N.
+config ARM_PV_FIXUP
+ def_bool y
+ depends on ARM_LPAE && ARM_PATCH_PHYS_VIRT && ARCH_KEYSTONE
+
config ARCH_PHYS_ADDR_T_64BIT
def_bool ARM_LPAE
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index d3afdf9eb65a..57c8df500e8c 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -18,6 +18,7 @@ obj-$(CONFIG_MODULES) += proc-syms.o
obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o
obj-$(CONFIG_HIGHMEM) += highmem.o
obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
+obj-$(CONFIG_ARM_PV_FIXUP) += pv-fixup-asm.o
obj-$(CONFIG_CPU_ABRT_NOMMU) += abort-nommu.o
obj-$(CONFIG_CPU_ABRT_EV4) += abort-ev4.o
@@ -55,6 +56,8 @@ obj-$(CONFIG_CPU_XSCALE) += copypage-xscale.o
obj-$(CONFIG_CPU_XSC3) += copypage-xsc3.o
obj-$(CONFIG_CPU_COPY_FA) += copypage-fa.o
+CFLAGS_copypage-feroceon.o := -march=armv5te
+
obj-$(CONFIG_CPU_TLB_V4WT) += tlb-v4.o
obj-$(CONFIG_CPU_TLB_V4WB) += tlb-v4wb.o
obj-$(CONFIG_CPU_TLB_V4WBI) += tlb-v4wbi.o
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index d6f5256acc0d..1ced8a0f7a52 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1896,7 +1896,7 @@ struct dma_map_ops iommu_coherent_ops = {
* arm_iommu_attach_device function.
*/
struct dma_iommu_mapping *
-arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size)
+arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size)
{
unsigned int bits = size >> PAGE_SHIFT;
unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
@@ -1904,6 +1904,10 @@ arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size)
int extensions = 1;
int err = -ENOMEM;
+ /* currently only 32-bit DMA address space is supported */
+ if (size > DMA_BIT_MASK(32) + 1)
+ return ERR_PTR(-ERANGE);
+
if (!bitmap_size)
return ERR_PTR(-EINVAL);
@@ -2075,13 +2079,6 @@ static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
if (!iommu)
return false;
- /*
- * currently arm_iommu_create_mapping() takes a max of size_t
- * for size param. So check this limit for now.
- */
- if (size > SIZE_MAX)
- return false;
-
mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
if (IS_ERR(mapping)) {
pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 4e6ef896c619..6ca7d9aa896f 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -1112,22 +1112,22 @@ void __init sanity_check_meminfo(void)
}
/*
- * Find the first non-section-aligned page, and point
+ * Find the first non-pmd-aligned page, and point
* memblock_limit at it. This relies on rounding the
- * limit down to be section-aligned, which happens at
- * the end of this function.
+ * limit down to be pmd-aligned, which happens at the
+ * end of this function.
*
* With this algorithm, the start or end of almost any
- * bank can be non-section-aligned. The only exception
- * is that the start of the bank 0 must be section-
+ * bank can be non-pmd-aligned. The only exception is
+ * that the start of the bank 0 must be section-
* aligned, since otherwise memory would need to be
* allocated when mapping the start of bank 0, which
* occurs before any free memory is mapped.
*/
if (!memblock_limit) {
- if (!IS_ALIGNED(block_start, SECTION_SIZE))
+ if (!IS_ALIGNED(block_start, PMD_SIZE))
memblock_limit = block_start;
- else if (!IS_ALIGNED(block_end, SECTION_SIZE))
+ else if (!IS_ALIGNED(block_end, PMD_SIZE))
memblock_limit = arm_lowmem_limit;
}
@@ -1137,12 +1137,12 @@ void __init sanity_check_meminfo(void)
high_memory = __va(arm_lowmem_limit - 1) + 1;
/*
- * Round the memblock limit down to a section size. This
+ * Round the memblock limit down to a pmd size. This
* helps to ensure that we will allocate memory from the
- * last full section, which should be mapped.
+ * last full pmd, which should be mapped.
*/
if (memblock_limit)
- memblock_limit = round_down(memblock_limit, SECTION_SIZE);
+ memblock_limit = round_down(memblock_limit, PMD_SIZE);
if (!memblock_limit)
memblock_limit = arm_lowmem_limit;
@@ -1387,123 +1387,98 @@ static void __init map_lowmem(void)
}
}
-#ifdef CONFIG_ARM_LPAE
+#ifdef CONFIG_ARM_PV_FIXUP
+extern unsigned long __atags_pointer;
+typedef void pgtables_remap(long long offset, unsigned long pgd, void *bdata);
+pgtables_remap lpae_pgtables_remap_asm;
+
/*
* early_paging_init() recreates boot time page table setup, allowing machines
* to switch over to a high (>4G) address space on LPAE systems
*/
-void __init early_paging_init(const struct machine_desc *mdesc,
- struct proc_info_list *procinfo)
+void __init early_paging_init(const struct machine_desc *mdesc)
{
- pmdval_t pmdprot = procinfo->__cpu_mm_mmu_flags;
- unsigned long map_start, map_end;
- pgd_t *pgd0, *pgdk;
- pud_t *pud0, *pudk, *pud_start;
- pmd_t *pmd0, *pmdk;
- phys_addr_t phys;
- int i;
+ pgtables_remap *lpae_pgtables_remap;
+ unsigned long pa_pgd;
+ unsigned int cr, ttbcr;
+ long long offset;
+ void *boot_data;
- if (!(mdesc->init_meminfo))
+ if (!mdesc->pv_fixup)
return;
- /* remap kernel code and data */
- map_start = init_mm.start_code & PMD_MASK;
- map_end = ALIGN(init_mm.brk, PMD_SIZE);
+ offset = mdesc->pv_fixup();
+ if (offset == 0)
+ return;
- /* get a handle on things... */
- pgd0 = pgd_offset_k(0);
- pud_start = pud0 = pud_offset(pgd0, 0);
- pmd0 = pmd_offset(pud0, 0);
+ /*
+ * Get the address of the remap function in the 1:1 identity
+ * mapping setup by the early page table assembly code. We
+ * must get this prior to the pv update. The following barrier
+ * ensures that this is complete before we fixup any P:V offsets.
+ */
+ lpae_pgtables_remap = (pgtables_remap *)(unsigned long)__pa(lpae_pgtables_remap_asm);
+ pa_pgd = __pa(swapper_pg_dir);
+ boot_data = __va(__atags_pointer);
+ barrier();
- pgdk = pgd_offset_k(map_start);
- pudk = pud_offset(pgdk, map_start);
- pmdk = pmd_offset(pudk, map_start);
+ pr_info("Switching physical address space to 0x%08llx\n",
+ (u64)PHYS_OFFSET + offset);
- mdesc->init_meminfo();
+ /* Re-set the phys pfn offset, and the pv offset */
+ __pv_offset += offset;
+ __pv_phys_pfn_offset += PFN_DOWN(offset);
/* Run the patch stub to update the constants */
fixup_pv_table(&__pv_table_begin,
(&__pv_table_end - &__pv_table_begin) << 2);
/*
- * Cache cleaning operations for self-modifying code
- * We should clean the entries by MVA but running a
- * for loop over every pv_table entry pointer would
- * just complicate the code.
- */
- flush_cache_louis();
- dsb(ishst);
- isb();
-
- /*
- * FIXME: This code is not architecturally compliant: we modify
- * the mappings in-place, indeed while they are in use by this
- * very same code. This may lead to unpredictable behaviour of
- * the CPU.
- *
- * Even modifying the mappings in a separate page table does
- * not resolve this.
- *
- * The architecture strongly recommends that when a mapping is
- * changed, that it is changed by first going via an invalid
- * mapping and back to the new mapping. This is to ensure that
- * no TLB conflicts (caused by the TLB having more than one TLB
- * entry match a translation) can occur. However, doing that
- * here will result in unmapping the code we are running.
- */
- pr_warn("WARNING: unsafe modification of in-place page tables - tainting kernel\n");
- add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
-
- /*
- * Remap level 1 table. This changes the physical addresses
- * used to refer to the level 2 page tables to the high
- * physical address alias, leaving everything else the same.
- */
- for (i = 0; i < PTRS_PER_PGD; pud0++, i++) {
- set_pud(pud0,
- __pud(__pa(pmd0) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
- pmd0 += PTRS_PER_PMD;
- }
-
- /*
- * Remap the level 2 table, pointing the mappings at the high
- * physical address alias of these pages.
- */
- phys = __pa(map_start);
- do {
- *pmdk++ = __pmd(phys | pmdprot);
- phys += PMD_SIZE;
- } while (phys < map_end);
-
- /*
- * Ensure that the above updates are flushed out of the cache.
- * This is not strictly correct; on a system where the caches
- * are coherent with each other, but the MMU page table walks
- * may not be coherent, flush_cache_all() may be a no-op, and
- * this will fail.
+ * We changing not only the virtual to physical mapping, but also
+ * the physical addresses used to access memory. We need to flush
+ * all levels of cache in the system with caching disabled to
+ * ensure that all data is written back, and nothing is prefetched
+ * into the caches. We also need to prevent the TLB walkers
+ * allocating into the caches too. Note that this is ARMv7 LPAE
+ * specific.
*/
+ cr = get_cr();
+ set_cr(cr & ~(CR_I | CR_C));
+ asm("mrc p15, 0, %0, c2, c0, 2" : "=r" (ttbcr));
+ asm volatile("mcr p15, 0, %0, c2, c0, 2"
+ : : "r" (ttbcr & ~(3 << 8 | 3 << 10)));
flush_cache_all();
/*
- * Re-write the TTBR values to point them at the high physical
- * alias of the page tables. We expect __va() will work on
- * cpu_get_pgd(), which returns the value of TTBR0.
+ * Fixup the page tables - this must be in the idmap region as
+ * we need to disable the MMU to do this safely, and hence it
+ * needs to be assembly. It's fairly simple, as we're using the
+ * temporary tables setup by the initial assembly code.
*/
- cpu_switch_mm(pgd0, &init_mm);
- cpu_set_ttbr(1, __pa(pgd0) + TTBR1_OFFSET);
+ lpae_pgtables_remap(offset, pa_pgd, boot_data);
- /* Finally flush any stale TLB values. */
- local_flush_bp_all();
- local_flush_tlb_all();
+ /* Re-enable the caches and cacheable TLB walks */
+ asm volatile("mcr p15, 0, %0, c2, c0, 2" : : "r" (ttbcr));
+ set_cr(cr);
}
#else
-void __init early_paging_init(const struct machine_desc *mdesc,
- struct proc_info_list *procinfo)
+void __init early_paging_init(const struct machine_desc *mdesc)
{
- if (mdesc->init_meminfo)
- mdesc->init_meminfo();
+ long long offset;
+
+ if (!mdesc->pv_fixup)
+ return;
+
+ offset = mdesc->pv_fixup();
+ if (offset == 0)
+ return;
+
+ pr_crit("Physical address space modification is only to support Keystone2.\n");
+ pr_crit("Please enable ARM_LPAE and ARM_PATCH_PHYS_VIRT support to use this\n");
+ pr_crit("feature. Your kernel may crash now, have a good day.\n");
+ add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
}
#endif
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index a014dfacd5ca..afd7e05d95f1 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -304,15 +304,6 @@ void __init sanity_check_meminfo(void)
}
/*
- * early_paging_init() recreates boot time page table setup, allowing machines
- * to switch over to a high (>4G) address space on LPAE systems
- */
-void __init early_paging_init(const struct machine_desc *mdesc,
- struct proc_info_list *procinfo)
-{
-}
-
-/*
* paging_init() sets up the page tables, initialises the zone memory
* maps, and sets up the zero page, bad page and bad page tables.
*/
diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S
index aa0519eed698..774ef1323554 100644
--- a/arch/arm/mm/proc-arm1020.S
+++ b/arch/arm/mm/proc-arm1020.S
@@ -22,8 +22,6 @@
*
* These are the low level assembler for performing cache and TLB
* functions on the arm1020.
- *
- * CONFIG_CPU_ARM1020_CPU_IDLE -> nohlt
*/
#include <linux/linkage.h>
#include <linux/init.h>
diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S
index bff4c7f70fd6..ae3c27b71594 100644
--- a/arch/arm/mm/proc-arm1020e.S
+++ b/arch/arm/mm/proc-arm1020e.S
@@ -22,8 +22,6 @@
*
* These are the low level assembler for performing cache and TLB
* functions on the arm1020e.
- *
- * CONFIG_CPU_ARM1020_CPU_IDLE -> nohlt
*/
#include <linux/linkage.h>
#include <linux/init.h>
diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S
index ede8c54ab4aa..32a47cc19076 100644
--- a/arch/arm/mm/proc-arm925.S
+++ b/arch/arm/mm/proc-arm925.S
@@ -441,9 +441,6 @@ ENTRY(cpu_arm925_set_pte_ext)
.type __arm925_setup, #function
__arm925_setup:
mov r0, #0
-#if defined(CONFIG_CPU_ICACHE_STREAMING_DISABLE)
- orr r0,r0,#1 << 7
-#endif
/* Transparent on, D-cache clean & flush mode. See NOTE2 above */
orr r0,r0,#1 << 1 @ transparent mode on
diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S
index e494d6d6acbe..92e08bf37aad 100644
--- a/arch/arm/mm/proc-feroceon.S
+++ b/arch/arm/mm/proc-feroceon.S
@@ -602,7 +602,6 @@ __\name\()_proc_info:
PMD_SECT_AP_WRITE | \
PMD_SECT_AP_READ
initfn __feroceon_setup, __\name\()_proc_info
- .long __feroceon_setup
.long cpu_arch_name
.long cpu_elf_name
.long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP
diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S
index 6f2f8f3cb33c..c6141a5435c3 100644
--- a/arch/arm/mm/proc-v7-2level.S
+++ b/arch/arm/mm/proc-v7-2level.S
@@ -150,10 +150,10 @@ ENDPROC(cpu_v7_set_pte_ext)
* Macro for setting up the TTBRx and TTBCR registers.
* - \ttb0 and \ttb1 updated with the corresponding flags.
*/
- .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp
+ .macro v7_ttb_setup, zero, ttbr0l, ttbr0h, ttbr1, tmp
mcr p15, 0, \zero, c2, c0, 2 @ TTB control register
- ALT_SMP(orr \ttbr0, \ttbr0, #TTB_FLAGS_SMP)
- ALT_UP(orr \ttbr0, \ttbr0, #TTB_FLAGS_UP)
+ ALT_SMP(orr \ttbr0l, \ttbr0l, #TTB_FLAGS_SMP)
+ ALT_UP(orr \ttbr0l, \ttbr0l, #TTB_FLAGS_UP)
ALT_SMP(orr \ttbr1, \ttbr1, #TTB_FLAGS_SMP)
ALT_UP(orr \ttbr1, \ttbr1, #TTB_FLAGS_UP)
mcr p15, 0, \ttbr1, c2, c0, 1 @ load TTB1
diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S
index d3daed0ae0ad..5e5720e8bc5f 100644
--- a/arch/arm/mm/proc-v7-3level.S
+++ b/arch/arm/mm/proc-v7-3level.S
@@ -126,11 +126,10 @@ ENDPROC(cpu_v7_set_pte_ext)
* Macro for setting up the TTBRx and TTBCR registers.
* - \ttbr1 updated.
*/
- .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp
+ .macro v7_ttb_setup, zero, ttbr0l, ttbr0h, ttbr1, tmp
ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address
- mov \tmp, \tmp, lsr #ARCH_PGD_SHIFT
- cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET?
- mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register
+ cmp \ttbr1, \tmp, lsr #12 @ PHYS_OFFSET > PAGE_OFFSET?
+ mrc p15, 0, \tmp, c2, c0, 2 @ TTB control egister
orr \tmp, \tmp, #TTB_EAE
ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP)
ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP)
@@ -143,13 +142,10 @@ ENDPROC(cpu_v7_set_pte_ext)
*/
orrls \tmp, \tmp, #TTBR1_SIZE @ TTBCR.T1SZ
mcr p15, 0, \tmp, c2, c0, 2 @ TTBCR
- mov \tmp, \ttbr1, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits
- mov \ttbr1, \ttbr1, lsl #ARCH_PGD_SHIFT @ lower bits
+ mov \tmp, \ttbr1, lsr #20
+ mov \ttbr1, \ttbr1, lsl #12
addls \ttbr1, \ttbr1, #TTBR1_OFFSET
mcrr p15, 1, \ttbr1, \tmp, c2 @ load TTBR1
- mov \tmp, \ttbr0, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits
- mov \ttbr0, \ttbr0, lsl #ARCH_PGD_SHIFT @ lower bits
- mcrr p15, 0, \ttbr0, \tmp, c2 @ load TTBR0
.endm
/*
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 3d1054f11a8a..003190ae9cd8 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -252,6 +252,12 @@ ENDPROC(cpu_pj4b_do_resume)
* Initialise TLB, Caches, and MMU state ready to switch the MMU
* on. Return in r0 the new CP15 C1 control register setting.
*
+ * r1, r2, r4, r5, r9, r13 must be preserved - r13 is not a stack
+ * r4: TTBR0 (low word)
+ * r5: TTBR0 (high word if LPAE)
+ * r8: TTBR1
+ * r9: Main ID register
+ *
* This should be able to cover all ARMv7 cores.
*
* It is assumed that:
@@ -279,6 +285,78 @@ __v7_ca17mp_setup:
#endif
b __v7_setup
+/*
+ * Errata:
+ * r0, r10 available for use
+ * r1, r2, r4, r5, r9, r13: must be preserved
+ * r3: contains MIDR rX number in bits 23-20
+ * r6: contains MIDR rXpY as 8-bit XY number
+ * r9: MIDR
+ */
+__ca8_errata:
+#if defined(CONFIG_ARM_ERRATA_430973) && !defined(CONFIG_ARCH_MULTIPLATFORM)
+ teq r3, #0x00100000 @ only present in r1p*
+ mrceq p15, 0, r0, c1, c0, 1 @ read aux control register
+ orreq r0, r0, #(1 << 6) @ set IBE to 1
+ mcreq p15, 0, r0, c1, c0, 1 @ write aux control register
+#endif
+#ifdef CONFIG_ARM_ERRATA_458693
+ teq r6, #0x20 @ only present in r2p0
+ mrceq p15, 0, r0, c1, c0, 1 @ read aux control register
+ orreq r0, r0, #(1 << 5) @ set L1NEON to 1
+ orreq r0, r0, #(1 << 9) @ set PLDNOP to 1
+ mcreq p15, 0, r0, c1, c0, 1 @ write aux control register
+#endif
+#ifdef CONFIG_ARM_ERRATA_460075
+ teq r6, #0x20 @ only present in r2p0
+ mrceq p15, 1, r0, c9, c0, 2 @ read L2 cache aux ctrl register
+ tsteq r0, #1 << 22
+ orreq r0, r0, #(1 << 22) @ set the Write Allocate disable bit
+ mcreq p15, 1, r0, c9, c0, 2 @ write the L2 cache aux ctrl register
+#endif
+ b __errata_finish
+
+__ca9_errata:
+#ifdef CONFIG_ARM_ERRATA_742230
+ cmp r6, #0x22 @ only present up to r2p2
+ mrcle p15, 0, r0, c15, c0, 1 @ read diagnostic register
+ orrle r0, r0, #1 << 4 @ set bit #4
+ mcrle p15, 0, r0, c15, c0, 1 @ write diagnostic register
+#endif
+#ifdef CONFIG_ARM_ERRATA_742231
+ teq r6, #0x20 @ present in r2p0
+ teqne r6, #0x21 @ present in r2p1
+ teqne r6, #0x22 @ present in r2p2
+ mrceq p15, 0, r0, c15, c0, 1 @ read diagnostic register
+ orreq r0, r0, #1 << 12 @ set bit #12
+ orreq r0, r0, #1 << 22 @ set bit #22
+ mcreq p15, 0, r0, c15, c0, 1 @ write diagnostic register
+#endif
+#ifdef CONFIG_ARM_ERRATA_743622
+ teq r3, #0x00200000 @ only present in r2p*
+ mrceq p15, 0, r0, c15, c0, 1 @ read diagnostic register
+ orreq r0, r0, #1 << 6 @ set bit #6
+ mcreq p15, 0, r0, c15, c0, 1 @ write diagnostic register
+#endif
+#if defined(CONFIG_ARM_ERRATA_751472) && defined(CONFIG_SMP)
+ ALT_SMP(cmp r6, #0x30) @ present prior to r3p0
+ ALT_UP_B(1f)
+ mrclt p15, 0, r0, c15, c0, 1 @ read diagnostic register
+ orrlt r0, r0, #1 << 11 @ set bit #11
+ mcrlt p15, 0, r0, c15, c0, 1 @ write diagnostic register
+1:
+#endif
+ b __errata_finish
+
+__ca15_errata:
+#ifdef CONFIG_ARM_ERRATA_773022
+ cmp r6, #0x4 @ only present up to r0p4
+ mrcle p15, 0, r0, c1, c0, 1 @ read aux control register
+ orrle r0, r0, #1 << 1 @ disable loop buffer
+ mcrle p15, 0, r0, c1, c0, 1 @ write aux control register
+#endif
+ b __errata_finish
+
__v7_pj4b_setup:
#ifdef CONFIG_CPU_PJ4B
@@ -339,96 +417,38 @@ __v7_setup:
bl v7_flush_dcache_louis
ldmia r12, {r0-r5, r7, r9, r11, lr}
- mrc p15, 0, r0, c0, c0, 0 @ read main ID register
- and r10, r0, #0xff000000 @ ARM?
- teq r10, #0x41000000
- bne 3f
- and r5, r0, #0x00f00000 @ variant
- and r6, r0, #0x0000000f @ revision
- orr r6, r6, r5, lsr #20-4 @ combine variant and revision
- ubfx r0, r0, #4, #12 @ primary part number
+ and r0, r9, #0xff000000 @ ARM?
+ teq r0, #0x41000000
+ bne __errata_finish
+ and r3, r9, #0x00f00000 @ variant
+ and r6, r9, #0x0000000f @ revision
+ orr r6, r6, r3, lsr #20-4 @ combine variant and revision
+ ubfx r0, r9, #4, #12 @ primary part number
/* Cortex-A8 Errata */
ldr r10, =0x00000c08 @ Cortex-A8 primary part number
teq r0, r10
- bne 2f
-#if defined(CONFIG_ARM_ERRATA_430973) && !defined(CONFIG_ARCH_MULTIPLATFORM)
-
- teq r5, #0x00100000 @ only present in r1p*
- mrceq p15, 0, r10, c1, c0, 1 @ read aux control register
- orreq r10, r10, #(1 << 6) @ set IBE to 1
- mcreq p15, 0, r10, c1, c0, 1 @ write aux control register
-#endif
-#ifdef CONFIG_ARM_ERRATA_458693
- teq r6, #0x20 @ only present in r2p0
- mrceq p15, 0, r10, c1, c0, 1 @ read aux control register
- orreq r10, r10, #(1 << 5) @ set L1NEON to 1
- orreq r10, r10, #(1 << 9) @ set PLDNOP to 1
- mcreq p15, 0, r10, c1, c0, 1 @ write aux control register
-#endif
-#ifdef CONFIG_ARM_ERRATA_460075
- teq r6, #0x20 @ only present in r2p0
- mrceq p15, 1, r10, c9, c0, 2 @ read L2 cache aux ctrl register
- tsteq r10, #1 << 22
- orreq r10, r10, #(1 << 22) @ set the Write Allocate disable bit
- mcreq p15, 1, r10, c9, c0, 2 @ write the L2 cache aux ctrl register
-#endif
- b 3f
+ beq __ca8_errata
/* Cortex-A9 Errata */
-2: ldr r10, =0x00000c09 @ Cortex-A9 primary part number
+ ldr r10, =0x00000c09 @ Cortex-A9 primary part number
teq r0, r10
- bne 3f
-#ifdef CONFIG_ARM_ERRATA_742230
- cmp r6, #0x22 @ only present up to r2p2
- mrcle p15, 0, r10, c15, c0, 1 @ read diagnostic register
- orrle r10, r10, #1 << 4 @ set bit #4
- mcrle p15, 0, r10, c15, c0, 1 @ write diagnostic register
-#endif
-#ifdef CONFIG_ARM_ERRATA_742231
- teq r6, #0x20 @ present in r2p0
- teqne r6, #0x21 @ present in r2p1
- teqne r6, #0x22 @ present in r2p2
- mrceq p15, 0, r10, c15, c0, 1 @ read diagnostic register
- orreq r10, r10, #1 << 12 @ set bit #12
- orreq r10, r10, #1 << 22 @ set bit #22
- mcreq p15, 0, r10, c15, c0, 1 @ write diagnostic register
-#endif
-#ifdef CONFIG_ARM_ERRATA_743622
- teq r5, #0x00200000 @ only present in r2p*
- mrceq p15, 0, r10, c15, c0, 1 @ read diagnostic register
- orreq r10, r10, #1 << 6 @ set bit #6
- mcreq p15, 0, r10, c15, c0, 1 @ write diagnostic register
-#endif
-#if defined(CONFIG_ARM_ERRATA_751472) && defined(CONFIG_SMP)
- ALT_SMP(cmp r6, #0x30) @ present prior to r3p0
- ALT_UP_B(1f)
- mrclt p15, 0, r10, c15, c0, 1 @ read diagnostic register
- orrlt r10, r10, #1 << 11 @ set bit #11
- mcrlt p15, 0, r10, c15, c0, 1 @ write diagnostic register
-1:
-#endif
+ beq __ca9_errata
/* Cortex-A15 Errata */
-3: ldr r10, =0x00000c0f @ Cortex-A15 primary part number
+ ldr r10, =0x00000c0f @ Cortex-A15 primary part number
teq r0, r10
- bne 4f
+ beq __ca15_errata
-#ifdef CONFIG_ARM_ERRATA_773022
- cmp r6, #0x4 @ only present up to r0p4
- mrcle p15, 0, r10, c1, c0, 1 @ read aux control register
- orrle r10, r10, #1 << 1 @ disable loop buffer
- mcrle p15, 0, r10, c1, c0, 1 @ write aux control register
-#endif
-
-4: mov r10, #0
+__errata_finish:
+ mov r10, #0
mcr p15, 0, r10, c7, c5, 0 @ I+BTB cache invalidate
#ifdef CONFIG_MMU
mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs
- v7_ttb_setup r10, r4, r8, r5 @ TTBCR, TTBRx setup
- ldr r5, =PRRR @ PRRR
+ v7_ttb_setup r10, r4, r5, r8, r3 @ TTBCR, TTBRx setup
+ ldr r3, =PRRR @ PRRR
ldr r6, =NMRR @ NMRR
- mcr p15, 0, r5, c10, c2, 0 @ write PRRR
+ mcr p15, 0, r3, c10, c2, 0 @ write PRRR
mcr p15, 0, r6, c10, c2, 1 @ write NMRR
#endif
dsb @ Complete invalidations
@@ -437,22 +457,22 @@ __v7_setup:
and r0, r0, #(0xf << 12) @ ThumbEE enabled field
teq r0, #(1 << 12) @ check if ThumbEE is present
bne 1f
- mov r5, #0
- mcr p14, 6, r5, c1, c0, 0 @ Initialize TEEHBR to 0
+ mov r3, #0
+ mcr p14, 6, r3, c1, c0, 0 @ Initialize TEEHBR to 0
mrc p14, 6, r0, c0, c0, 0 @ load TEECR
orr r0, r0, #1 @ set the 1st bit in order to
mcr p14, 6, r0, c0, c0, 0 @ stop userspace TEEHBR access
1:
#endif
- adr r5, v7_crval
- ldmia r5, {r5, r6}
+ adr r3, v7_crval
+ ldmia r3, {r3, r6}
ARM_BE8(orr r6, r6, #1 << 25) @ big-endian page tables
#ifdef CONFIG_SWP_EMULATE
- orr r5, r5, #(1 << 10) @ set SW bit in "clear"
+ orr r3, r3, #(1 << 10) @ set SW bit in "clear"
bic r6, r6, #(1 << 10) @ clear it in "mmuset"
#endif
mrc p15, 0, r0, c1, c0, 0 @ read control register
- bic r0, r0, r5 @ clear bits them
+ bic r0, r0, r3 @ clear bits them
orr r0, r0, r6 @ set them
THUMB( orr r0, r0, #1 << 30 ) @ Thumb exceptions
ret lr @ return to head.S:__ret
diff --git a/arch/arm/mm/pv-fixup-asm.S b/arch/arm/mm/pv-fixup-asm.S
new file mode 100644
index 000000000000..1867f3e43016
--- /dev/null
+++ b/arch/arm/mm/pv-fixup-asm.S
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2015 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This assembly is required to safely remap the physical address space
+ * for Keystone 2
+ */
+#include <linux/linkage.h>
+#include <asm/asm-offsets.h>
+#include <asm/cp15.h>
+#include <asm/memory.h>
+#include <asm/pgtable.h>
+
+ .section ".idmap.text", "ax"
+
+#define L1_ORDER 3
+#define L2_ORDER 3
+
+ENTRY(lpae_pgtables_remap_asm)
+ stmfd sp!, {r4-r8, lr}
+
+ mrc p15, 0, r8, c1, c0, 0 @ read control reg
+ bic ip, r8, #CR_M @ disable caches and MMU
+ mcr p15, 0, ip, c1, c0, 0
+ dsb
+ isb
+
+ /* Update level 2 entries covering the kernel */
+ ldr r6, =(_end - 1)
+ add r7, r2, #0x1000
+ add r6, r7, r6, lsr #SECTION_SHIFT - L2_ORDER
+ add r7, r7, #PAGE_OFFSET >> (SECTION_SHIFT - L2_ORDER)
+1: ldrd r4, [r7]
+ adds r4, r4, r0
+ adc r5, r5, r1
+ strd r4, [r7], #1 << L2_ORDER
+ cmp r7, r6
+ bls 1b
+
+ /* Update level 2 entries for the boot data */
+ add r7, r2, #0x1000
+ add r7, r7, r3, lsr #SECTION_SHIFT - L2_ORDER
+ bic r7, r7, #(1 << L2_ORDER) - 1
+ ldrd r4, [r7]
+ adds r4, r4, r0
+ adc r5, r5, r1
+ strd r4, [r7], #1 << L2_ORDER
+ ldrd r4, [r7]
+ adds r4, r4, r0
+ adc r5, r5, r1
+ strd r4, [r7]
+
+ /* Update level 1 entries */
+ mov r6, #4
+ mov r7, r2
+2: ldrd r4, [r7]
+ adds r4, r4, r0
+ adc r5, r5, r1
+ strd r4, [r7], #1 << L1_ORDER
+ subs r6, r6, #1
+ bne 2b
+
+ mrrc p15, 0, r4, r5, c2 @ read TTBR0
+ adds r4, r4, r0 @ update physical address
+ adc r5, r5, r1
+ mcrr p15, 0, r4, r5, c2 @ write back TTBR0
+ mrrc p15, 1, r4, r5, c2 @ read TTBR1
+ adds r4, r4, r0 @ update physical address
+ adc r5, r5, r1
+ mcrr p15, 1, r4, r5, c2 @ write back TTBR1
+
+ dsb
+
+ mov ip, #0
+ mcr p15, 0, ip, c7, c5, 0 @ I+BTB cache invalidate
+ mcr p15, 0, ip, c8, c7, 0 @ local_flush_tlb_all()
+ dsb
+ isb
+
+ mcr p15, 0, r8, c1, c0, 0 @ re-enable MMU
+ dsb
+ isb
+
+ ldmfd sp!, {r4-r8, pc}
+ENDPROC(lpae_pgtables_remap_asm)
OpenPOWER on IntegriCloud