summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/mmzone.h4
-rw-r--r--mm/page_alloc.c36
2 files changed, 33 insertions, 7 deletions
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 4721e9aa3ced..a8140a9a65e8 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -37,11 +37,13 @@
#define MIGRATE_UNMOVABLE 0
#define MIGRATE_RECLAIMABLE 1
#define MIGRATE_MOVABLE 2
-#define MIGRATE_TYPES 3
+#define MIGRATE_HIGHATOMIC 3
+#define MIGRATE_TYPES 4
#else
#define MIGRATE_UNMOVABLE 0
#define MIGRATE_UNRECLAIMABLE 0
#define MIGRATE_MOVABLE 0
+#define MIGRATE_HIGHATOMIC 0
#define MIGRATE_TYPES 1
#endif
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 29f4de1423c9..03fef8d987f6 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -170,10 +170,16 @@ static void set_pageblock_migratetype(struct page *page, int migratetype)
PB_migrate, PB_migrate_end);
}
-static inline int gfpflags_to_migratetype(gfp_t gfp_flags)
+static inline int allocflags_to_migratetype(gfp_t gfp_flags, int order)
{
WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
+ /* Cluster high-order atomic allocations together */
+ if (unlikely(order > 0) &&
+ (!(gfp_flags & __GFP_WAIT) || in_interrupt()))
+ return MIGRATE_HIGHATOMIC;
+
+ /* Cluster based on mobility */
return (((gfp_flags & __GFP_MOVABLE) != 0) << 1) |
((gfp_flags & __GFP_RECLAIMABLE) != 0);
}
@@ -188,7 +194,7 @@ static void set_pageblock_migratetype(struct page *page, int migratetype)
{
}
-static inline int gfpflags_to_migratetype(gfp_t gfp_flags)
+static inline int allocflags_to_migratetype(gfp_t gfp_flags, int order)
{
return MIGRATE_UNMOVABLE;
}
@@ -679,9 +685,10 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
* the free lists for the desirable migrate type are depleted
*/
static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
- [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE },
- [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE },
- [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE },
+ [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_HIGHATOMIC },
+ [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_HIGHATOMIC },
+ [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE,MIGRATE_HIGHATOMIC },
+ [MIGRATE_HIGHATOMIC] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE,MIGRATE_MOVABLE},
};
/*
@@ -758,13 +765,24 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
int current_order;
struct page *page;
int migratetype, i;
+ int nonatomic_fallback_atomic = 0;
+retry:
/* Find the largest possible block of pages in the other list */
for (current_order = MAX_ORDER-1; current_order >= order;
--current_order) {
for (i = 0; i < MIGRATE_TYPES - 1; i++) {
migratetype = fallbacks[start_migratetype][i];
+ /*
+ * Make it hard to fallback to blocks used for
+ * high-order atomic allocations
+ */
+ if (migratetype == MIGRATE_HIGHATOMIC &&
+ start_migratetype != MIGRATE_UNMOVABLE &&
+ !nonatomic_fallback_atomic)
+ continue;
+
area = &(zone->free_area[current_order]);
if (list_empty(&area->free_list[migratetype]))
continue;
@@ -797,6 +815,12 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
}
}
+ /* Allow fallback to high-order atomic blocks if memory is that low */
+ if (!nonatomic_fallback_atomic) {
+ nonatomic_fallback_atomic = 1;
+ goto retry;
+ }
+
return NULL;
}
#else
@@ -1058,7 +1082,7 @@ static struct page *buffered_rmqueue(struct zonelist *zonelist,
struct page *page;
int cold = !!(gfp_flags & __GFP_COLD);
int cpu;
- int migratetype = gfpflags_to_migratetype(gfp_flags);
+ int migratetype = allocflags_to_migratetype(gfp_flags, order);
again:
cpu = get_cpu();
OpenPOWER on IntegriCloud