From f048aace29e007f2b642097e2da8231e0e9cce2d Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Thu, 18 Dec 2008 19:13:38 +0000 Subject: powerpc/mm: Add SMP support to no-hash TLB handling This commit moves the whole no-hash TLB handling out of line into a new tlb_nohash.c file, and implements some basic SMP support using IPIs and/or broadcast tlbivax instructions. Note that I'm using local invalidations for D->I cache coherency. At worst, if another processor is trying to execute the same and has the old entry in its TLB, it will just take a fault and re-do the TLB flush locally (it won't re-do the cache flush in any case). Signed-off-by: Benjamin Herrenschmidt Acked-by: Kumar Gala Signed-off-by: Paul Mackerras --- arch/powerpc/include/asm/highmem.h | 4 +- arch/powerpc/include/asm/mmu.h | 16 +++++++ arch/powerpc/include/asm/tlbflush.h | 84 +++++++++++++++++-------------------- 3 files changed, 56 insertions(+), 48 deletions(-) (limited to 'arch/powerpc/include') diff --git a/arch/powerpc/include/asm/highmem.h b/arch/powerpc/include/asm/highmem.h index 7dc52eca8b67..fd97e501aa6a 100644 --- a/arch/powerpc/include/asm/highmem.h +++ b/arch/powerpc/include/asm/highmem.h @@ -85,7 +85,7 @@ static inline void *kmap_atomic_prot(struct page *page, enum km_type type, pgpro BUG_ON(!pte_none(*(kmap_pte-idx))); #endif __set_pte_at(&init_mm, vaddr, kmap_pte-idx, mk_pte(page, prot)); - local_flush_tlb_page(vaddr); + local_flush_tlb_page(NULL, vaddr); return (void*) vaddr; } @@ -113,7 +113,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type) * this pte without first remap it */ pte_clear(&init_mm, vaddr, kmap_pte-idx); - local_flush_tlb_page(vaddr); + local_flush_tlb_page(NULL, vaddr); #endif pagefault_enable(); } diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index dc8c0aef5e6c..6e7639911318 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -30,6 +30,22 @@ */ #define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000) +/* Enable use of broadcast TLB invalidations. We don't always set it + * on processors that support it due to other constraints with the + * use of such invalidations + */ +#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000) + +/* Enable use of tlbilx invalidate-by-PID variant. + */ +#define MMU_FTR_USE_TLBILX_PID ASM_CONST(0x00080000) + +/* This indicates that the processor cannot handle multiple outstanding + * broadcast tlbivax or tlbsync. This makes the code use a spinlock + * around such invalidate forms. + */ +#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000) + #ifndef __ASSEMBLY__ #include diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h index 9ed363d3de44..8c39b27c1ed7 100644 --- a/arch/powerpc/include/asm/tlbflush.h +++ b/arch/powerpc/include/asm/tlbflush.h @@ -6,7 +6,9 @@ * * - flush_tlb_mm(mm) flushes the specified mm context TLB's * - flush_tlb_page(vma, vmaddr) flushes one page - * - local_flush_tlb_page(vmaddr) flushes one page on the local processor + * - local_flush_tlb_mm(mm) flushes the specified mm context on + * the local processor + * - local_flush_tlb_page(vma, vmaddr) flushes one page on the local processor * - flush_tlb_page_nohash(vma, vmaddr) flushes one page if SW loaded TLB * - flush_tlb_range(vma, start, end) flushes a range of pages * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages @@ -18,7 +20,7 @@ */ #ifdef __KERNEL__ -#if defined(CONFIG_4xx) || defined(CONFIG_8xx) || defined(CONFIG_FSL_BOOKE) +#ifdef CONFIG_PPC_MMU_NOHASH /* * TLB flushing for software loaded TLB chips * @@ -31,10 +33,10 @@ #define MMU_NO_CONTEXT ((unsigned int)-1) -extern void _tlbie(unsigned long address, unsigned int pid); extern void _tlbil_all(void); extern void _tlbil_pid(unsigned int pid); extern void _tlbil_va(unsigned long address, unsigned int pid); +extern void _tlbivax_bcast(unsigned long address, unsigned int pid); #if defined(CONFIG_40x) || defined(CONFIG_8xx) #define _tlbia() asm volatile ("tlbia; sync" : : : "memory") @@ -42,48 +44,26 @@ extern void _tlbil_va(unsigned long address, unsigned int pid); extern void _tlbia(void); #endif -static inline void local_flush_tlb_mm(struct mm_struct *mm) -{ - _tlbil_pid(mm->context.id); -} - -static inline void flush_tlb_mm(struct mm_struct *mm) -{ - _tlbil_pid(mm->context.id); -} - -static inline void local_flush_tlb_page(unsigned long vmaddr) -{ - _tlbil_va(vmaddr, 0); -} - -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long vmaddr) -{ - _tlbil_va(vmaddr, vma ? vma->vm_mm->context.id : 0); -} +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); -static inline void flush_tlb_page_nohash(struct vm_area_struct *vma, - unsigned long vmaddr) -{ - flush_tlb_page(vma, vmaddr); -} +extern void local_flush_tlb_mm(struct mm_struct *mm); +extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); -static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - _tlbil_pid(vma->vm_mm->context.id); -} +#ifdef CONFIG_SMP +extern void flush_tlb_mm(struct mm_struct *mm); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +#else +#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) +#define flush_tlb_page(vma,addr) local_flush_tlb_page(vma,addr) +#endif +#define flush_tlb_page_nohash(vma,addr) flush_tlb_page(vma,addr) -static inline void flush_tlb_kernel_range(unsigned long start, - unsigned long end) -{ - _tlbil_pid(0); -} +#elif defined(CONFIG_PPC_STD_MMU_32) -#elif defined(CONFIG_PPC32) /* - * TLB flushing for "classic" hash-MMMU 32-bit CPUs, 6xx, 7xx, 7xxx + * TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx */ extern void _tlbie(unsigned long address); extern void _tlbia(void); @@ -94,14 +74,20 @@ extern void flush_tlb_page_nohash(struct vm_area_struct *vma, unsigned long addr extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); -static inline void local_flush_tlb_page(unsigned long vmaddr) +static inline void local_flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) { - flush_tlb_page(NULL, vmaddr); + flush_tlb_page(vma, vmaddr); +} +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ + flush_tlb_mm(mm); } -#else +#elif defined(CONFIG_PPC_STD_MMU_64) + /* - * TLB flushing for 64-bit has-MMU CPUs + * TLB flushing for 64-bit hash-MMU CPUs */ #include @@ -151,11 +137,16 @@ extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize, extern void flush_hash_range(unsigned long number, int local); +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ +} + static inline void flush_tlb_mm(struct mm_struct *mm) { } -static inline void local_flush_tlb_page(unsigned long vmaddr) +static inline void local_flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) { } @@ -183,7 +174,8 @@ static inline void flush_tlb_kernel_range(unsigned long start, extern void __flush_hash_table_range(struct mm_struct *mm, unsigned long start, unsigned long end); - +#else +#error Unsupported MMU type #endif #endif /*__KERNEL__ */ -- cgit v1.2.1