diff options
| -rw-r--r-- | include/linux/mmzone.h | 4 | ||||
| -rw-r--r-- | mm/page_alloc.c | 36 |
2 files changed, 33 insertions, 7 deletions
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 4721e9aa3ced..a8140a9a65e8 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -37,11 +37,13 @@ #define MIGRATE_UNMOVABLE 0 #define MIGRATE_RECLAIMABLE 1 #define MIGRATE_MOVABLE 2 -#define MIGRATE_TYPES 3 +#define MIGRATE_HIGHATOMIC 3 +#define MIGRATE_TYPES 4 #else #define MIGRATE_UNMOVABLE 0 #define MIGRATE_UNRECLAIMABLE 0 #define MIGRATE_MOVABLE 0 +#define MIGRATE_HIGHATOMIC 0 #define MIGRATE_TYPES 1 #endif diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 29f4de1423c9..03fef8d987f6 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -170,10 +170,16 @@ static void set_pageblock_migratetype(struct page *page, int migratetype) PB_migrate, PB_migrate_end); } -static inline int gfpflags_to_migratetype(gfp_t gfp_flags) +static inline int allocflags_to_migratetype(gfp_t gfp_flags, int order) { WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); + /* Cluster high-order atomic allocations together */ + if (unlikely(order > 0) && + (!(gfp_flags & __GFP_WAIT) || in_interrupt())) + return MIGRATE_HIGHATOMIC; + + /* Cluster based on mobility */ return (((gfp_flags & __GFP_MOVABLE) != 0) << 1) | ((gfp_flags & __GFP_RECLAIMABLE) != 0); } @@ -188,7 +194,7 @@ static void set_pageblock_migratetype(struct page *page, int migratetype) { } -static inline int gfpflags_to_migratetype(gfp_t gfp_flags) +static inline int allocflags_to_migratetype(gfp_t gfp_flags, int order) { return MIGRATE_UNMOVABLE; } @@ -679,9 +685,10 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags) * the free lists for the desirable migrate type are depleted */ static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = { - [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE }, - [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE }, - [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE }, + [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_HIGHATOMIC }, + [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_HIGHATOMIC }, + [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE,MIGRATE_HIGHATOMIC }, + [MIGRATE_HIGHATOMIC] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE,MIGRATE_MOVABLE}, }; /* @@ -758,13 +765,24 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order, int current_order; struct page *page; int migratetype, i; + int nonatomic_fallback_atomic = 0; +retry: /* Find the largest possible block of pages in the other list */ for (current_order = MAX_ORDER-1; current_order >= order; --current_order) { for (i = 0; i < MIGRATE_TYPES - 1; i++) { migratetype = fallbacks[start_migratetype][i]; + /* + * Make it hard to fallback to blocks used for + * high-order atomic allocations + */ + if (migratetype == MIGRATE_HIGHATOMIC && + start_migratetype != MIGRATE_UNMOVABLE && + !nonatomic_fallback_atomic) + continue; + area = &(zone->free_area[current_order]); if (list_empty(&area->free_list[migratetype])) continue; @@ -797,6 +815,12 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order, } } + /* Allow fallback to high-order atomic blocks if memory is that low */ + if (!nonatomic_fallback_atomic) { + nonatomic_fallback_atomic = 1; + goto retry; + } + return NULL; } #else @@ -1058,7 +1082,7 @@ static struct page *buffered_rmqueue(struct zonelist *zonelist, struct page *page; int cold = !!(gfp_flags & __GFP_COLD); int cpu; - int migratetype = gfpflags_to_migratetype(gfp_flags); + int migratetype = allocflags_to_migratetype(gfp_flags, order); again: cpu = get_cpu(); |
