3 files changed, 21 insertions, 18 deletions

diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index 38149b63dc44..7abf318b1522 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -35,7 +35,7 @@ static inline int init_new_context(struct task_struct *tsk,
 #define LCTL_OPCODE "lctlg"
 #endif
 
-static inline void update_mm(struct mm_struct *mm, struct task_struct *tsk)
+static inline void update_user_asce(struct mm_struct *mm)
 {
 	pgd_t *pgd = mm->pgd;
 
@@ -45,6 +45,13 @@ static inline void update_mm(struct mm_struct *mm, struct task_struct *tsk)
 	set_fs(current->thread.mm_segment);
 }
 
+static inline void clear_user_asce(struct mm_struct *mm)
+{
+	S390_lowcore.user_asce = S390_lowcore.kernel_asce;
+	asm volatile(LCTL_OPCODE" 1,1,%0\n" : : "m" (S390_lowcore.user_asce));
+	asm volatile(LCTL_OPCODE" 7,7,%0\n" : : "m" (S390_lowcore.user_asce));
+}
+
 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 			     struct task_struct *tsk)
 {
@@ -53,11 +60,13 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	if (prev == next)
 		return;
 	if (atomic_inc_return(&next->context.attach_count) >> 16) {
-		/* Delay update_mm until all TLB flushes are done. */
+		/* Delay update_user_asce until all TLB flushes are done. */
 		set_tsk_thread_flag(tsk, TIF_TLB_WAIT);
+		/* Clear old ASCE by loading the kernel ASCE. */
+		clear_user_asce(next);
 	} else {
 		cpumask_set_cpu(cpu, mm_cpumask(next));
-		update_mm(next, tsk);
+		update_user_asce(next);
 		if (next->context.flush_mm)
 			/* Flush pending TLBs */
 			__tlb_flush_mm(next);
@@ -80,7 +89,7 @@ static inline void finish_arch_post_lock_switch(void)
 		cpu_relax();
 
 	cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
-	update_mm(mm, tsk);
+	update_user_asce(mm);
 	if (mm->context.flush_mm)
 		__tlb_flush_mm(mm);
 	preempt_enable();
diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 2cb846c4b37f..c544b6f05d95 100644
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -57,8 +57,6 @@ static inline void tlb_gather_mmu(struct mmu_gather *tlb,
 	tlb->end = end;
 	tlb->fullmm = !(start | (end+1));
 	tlb->batch = NULL;
-	if (tlb->fullmm)
-		__tlb_flush_mm(mm);
 }
 
 static inline void tlb_flush_mmu(struct mmu_gather *tlb)
@@ -96,9 +94,7 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 				unsigned long address)
 {
-	if (!tlb->fullmm)
-		return page_table_free_rcu(tlb, (unsigned long *) pte);
-	page_table_free(tlb->mm, (unsigned long *) pte);
+	page_table_free_rcu(tlb, (unsigned long *) pte);
 }
 
 /*
@@ -114,9 +110,7 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 31))
 		return;
-	if (!tlb->fullmm)
-		return tlb_remove_table(tlb, pmd);
-	crst_table_free(tlb->mm, (unsigned long *) pmd);
+	tlb_remove_table(tlb, pmd);
 #endif
 }
 
@@ -133,9 +127,7 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 42))
 		return;
-	if (!tlb->fullmm)
-		return tlb_remove_table(tlb, pud);
-	crst_table_free(tlb->mm, (unsigned long *) pud);
+	tlb_remove_table(tlb, pud);
 #endif
 }
 
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 796c9320c709..24c62900b532 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -54,7 +54,7 @@ static void __crst_table_upgrade(void *arg)
 	struct mm_struct *mm = arg;
 
 	if (current->active_mm == mm)
-		update_mm(mm, current);
+		update_user_asce(mm);
 	__tlb_flush_local();
 }
 
@@ -107,8 +107,10 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
 {
 	pgd_t *pgd;
 
-	if (current->active_mm == mm)
+	if (current->active_mm == mm) {
+		clear_user_asce(mm);
 		__tlb_flush_mm(mm);
+	}
 	while (mm->context.asce_limit > limit) {
 		pgd = mm->pgd;
 		switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) {
@@ -132,7 +134,7 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
 		crst_table_free(mm, (unsigned long *) pgd);
 	}
 	if (current->active_mm == mm)
-		update_mm(mm, current);
+		update_user_asce(mm);
 }
 #endif