x86: cpa, preserve large pages if possible

When CPA is called on a range which fits into a large page mapping,
avoid to split the page when:

1) There is no change of attributes
2) The range to change is a complete large mapping

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 130 insertions, 12 deletions

diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 79a9f1b42ddd..40b7ac58e671 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -18,12 +18,17 @@
 
 struct cpa_data {
 	unsigned long	vaddr;
-	int		numpages;
 	pgprot_t	mask_set;
 	pgprot_t	mask_clr;
+	int		numpages;
 	int		flushtlb;
 };
 
+enum {
+	CPA_NO_SPLIT = 0,
+	CPA_SPLIT,
+};
+
 static inline int
 within(unsigned long addr, unsigned long start, unsigned long end)
 {
@@ -230,6 +235,86 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
 #endif
 }
 
+static int try_preserve_large_page(pte_t *kpte, unsigned long address,
+				   struct cpa_data *cpa)
+{
+	unsigned long nextpage_addr, numpages, pmask, psize, flags;
+	pte_t new_pte, old_pte, *tmp;
+	pgprot_t old_prot, new_prot;
+	int level, res = CPA_SPLIT;
+
+	spin_lock_irqsave(&pgd_lock, flags);
+	/*
+	 * Check for races, another CPU might have split this page
+	 * up already:
+	 */
+	tmp = lookup_address(address, &level);
+	if (tmp != kpte)
+		goto out_unlock;
+
+	switch (level) {
+	case PG_LEVEL_2M:
+		psize = LARGE_PAGE_SIZE;
+		pmask = LARGE_PAGE_MASK;
+		break;
+	case PG_LEVEL_1G:
+	default:
+		res = -EINVAL;
+		goto out_unlock;
+	}
+
+	/*
+	 * Calculate the number of pages, which fit into this large
+	 * page starting at address:
+	 */
+	nextpage_addr = (address + psize) & pmask;
+	numpages = (nextpage_addr - address) >> PAGE_SHIFT;
+	if (numpages < cpa->numpages)
+		cpa->numpages = numpages;
+
+	/*
+	 * We are safe now. Check whether the new pgprot is the same:
+	 */
+	old_pte = *kpte;
+	old_prot = new_prot = pte_pgprot(old_pte);
+
+	pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
+	pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
+	new_prot = static_protections(new_prot, address);
+
+	/*
+	 * If there are no changes, return. maxpages has been updated
+	 * above:
+	 */
+	if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
+		res = CPA_NO_SPLIT;
+		goto out_unlock;
+	}
+
+	/*
+	 * We need to change the attributes. Check, whether we can
+	 * change the large page in one go. We request a split, when
+	 * the address is not aligned and the number of pages is
+	 * smaller than the number of pages in the large page. Note
+	 * that we limited the number of possible pages already to
+	 * the number of pages in the large page.
+	 */
+	if (address == (nextpage_addr - psize) && cpa->numpages == numpages) {
+		/*
+		 * The address is aligned and the number of pages
+		 * covers the full page.
+		 */
+		new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot));
+		__set_pmd_pte(kpte, address, new_pte);
+		cpa->flushtlb = 1;
+		res = CPA_NO_SPLIT;
+	}
+
+out_unlock:
+	spin_unlock_irqrestore(&pgd_lock, flags);
+	return res;
+}
+
 static int split_large_page(pte_t *kpte, unsigned long address)
 {
 	pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
@@ -295,7 +380,7 @@ out_unlock:
 static int __change_page_attr(unsigned long address, struct cpa_data *cpa)
 {
 	struct page *kpte_page;
-	int level, err = 0;
+	int level, res;
 	pte_t *kpte;
 
 repeat:
@@ -338,13 +423,34 @@ repeat:
 			set_pte_atomic(kpte, new_pte);
 			cpa->flushtlb = 1;
 		}
-	} else {
-		err = split_large_page(kpte, address);
-		if (!err)
-			goto repeat;
-		cpa->flushtlb = 1;
+		cpa->numpages = 1;
+		return 0;
 	}
-	return err;
+
+	/*
+	 * Check, whether we can keep the large page intact
+	 * and just change the pte:
+	 */
+	res = try_preserve_large_page(kpte, address, cpa);
+	if (res < 0)
+		return res;
+
+	/*
+	 * When the range fits into the existing large page,
+	 * return. cp->numpages and cpa->tlbflush have been updated in
+	 * try_large_page:
+	 */
+	if (res == CPA_NO_SPLIT)
+		return 0;
+
+	/*
+	 * We have to split the large page:
+	 */
+	res = split_large_page(kpte, address);
+	if (res)
+		return res;
+	cpa->flushtlb = 1;
+	goto repeat;
 }
 
 /**
@@ -410,15 +516,27 @@ static int change_page_attr_addr(struct cpa_data *cpa)
 
 static int __change_page_attr_set_clr(struct cpa_data *cpa)
 {
-	unsigned int i;
-	int ret;
+	int ret, numpages = cpa->numpages;
 
-	for (i = 0; i < cpa->numpages ; i++, cpa->vaddr += PAGE_SIZE) {
+	while (numpages) {
+		/*
+		 * Store the remaining nr of pages for the large page
+		 * preservation check.
+		 */
+		cpa->numpages = numpages;
 		ret = change_page_attr_addr(cpa);
 		if (ret)
 			return ret;
-	}
 
+		/*
+		 * Adjust the number of pages with the result of the
+		 * CPA operation. Either a large page has been
+		 * preserved or a single page update happened.
+		 */
+		BUG_ON(cpa->numpages > numpages);
+		numpages -= cpa->numpages;
+		cpa->vaddr += cpa->numpages * PAGE_SIZE;
+	}
 	return 0;
 }