drm: Improve drm_mm search (and fix topdown allocation) with rbtrees

The drm_mm range manager claimed to support top-down insertion, but it
was neither searching for the top-most hole that could fit the
allocation request nor fitting the request to the hole correctly.

In order to search the range efficiently, we create a secondary index
for the holes using either their size or their address. This index
allows us to find the smallest hole or the hole at the bottom or top of
the range efficiently, whilst keeping the hole stack to rapidly service
evictions.

v2: Search for holes both high and low. Rename flags to mode.
v3: Discover rb_entry_safe() and use it!
v4: Kerneldoc for enum drm_mm_insert_mode.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Alex Deucher <alexander.deucher@amd.com>
Cc: "Christian König" <christian.koenig@amd.com>
Cc: David Airlie <airlied@linux.ie>
Cc: Russell King <rmk+kernel@armlinux.org.uk>
Cc: Daniel Vetter <daniel.vetter@intel.com>
Cc: Jani Nikula <jani.nikula@linux.intel.com>
Cc: Sean Paul <seanpaul@chromium.org>
Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Rob Clark <robdclark@gmail.com>
Cc: Thierry Reding <thierry.reding@gmail.com>
Cc: Stephen Warren <swarren@wwwdotorg.org>
Cc: Alexandre Courbot <gnurou@gmail.com>
Cc: Eric Anholt <eric@anholt.net>
Cc: Sinclair Yeh <syeh@vmware.com>
Cc: Thomas Hellstrom <thellstrom@vmware.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Reviewed-by: Sinclair Yeh <syeh@vmware.com> # vmwgfx
Reviewed-by: Lucas Stach <l.stach@pengutronix.de> #etnaviv
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/20170202210438.28702-1-chris@chris-wilson.co.uk

1 files changed, 92 insertions, 92 deletions

diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h
index 3bddca8fd2b5..d81b0ba9921f 100644
--- a/include/drm/drm_mm.h
+++ b/include/drm/drm_mm.h
@@ -53,19 +53,62 @@
 #define DRM_MM_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
 #endif
 
-enum drm_mm_search_flags {
-	DRM_MM_SEARCH_DEFAULT =		0,
-	DRM_MM_SEARCH_BEST =		1 << 0,
-	DRM_MM_SEARCH_BELOW =		1 << 1,
-};
+/**
+ * enum drm_mm_insert_mode - control search and allocation behaviour
+ *
+ * The &struct drm_mm range manager supports finding a suitable modes using
+ * a number of search trees. These trees are oranised by size, by address and
+ * in most recent eviction order. This allows the user to find either the
+ * smallest hole to reuse, the lowest or highest address to reuse, or simply
+ * reuse the most recent eviction that fits. When allocating the &drm_mm_node
+ * from within the hole, the &drm_mm_insert_mode also dictate whether to
+ * allocate the lowest matching address or the highest.
+ */
+enum drm_mm_insert_mode {
+	/**
+	 * @DRM_MM_INSERT_BEST:
+	 *
+	 * Search for the smallest hole (within the search range) that fits
+	 * the desired node.
+	 *
+	 * Allocates the node from the bottom of the found hole.
+	 */
+	DRM_MM_INSERT_BEST = 0,
 
-enum drm_mm_allocator_flags {
-	DRM_MM_CREATE_DEFAULT =		0,
-	DRM_MM_CREATE_TOP =		1 << 0,
-};
+	/**
+	 * @DRM_MM_INSERT_LOW:
+	 *
+	 * Search for the lowest hole (address closest to 0, within the search
+	 * range) that fits the desired node.
+	 *
+	 * Allocates the node from the bottom of the found hole.
+	 */
+	DRM_MM_INSERT_LOW,
 
-#define DRM_MM_BOTTOMUP DRM_MM_SEARCH_DEFAULT, DRM_MM_CREATE_DEFAULT
-#define DRM_MM_TOPDOWN DRM_MM_SEARCH_BELOW, DRM_MM_CREATE_TOP
+	/**
+	 * @DRM_MM_INSERT_HIGH:
+	 *
+	 * Search for the highest hole (address closest to U64_MAX, within the
+	 * search range) that fits the desired node.
+	 *
+	 * Allocates the node from the *top* of the found hole. The specified
+	 * alignment for the node is applied to the base of the node
+	 * (&drm_mm_node.start).
+	 */
+	DRM_MM_INSERT_HIGH,
+
+	/**
+	 * @DRM_MM_INSERT_EVICT:
+	 *
+	 * Search for the most recently evicted hole (within the search range)
+	 * that fits the desired node. This is appropriate for use immediately
+	 * after performing an eviction scan (see drm_mm_scan_init()) and
+	 * removing the selected nodes to form a hole.
+	 *
+	 * Allocates the node from the bottom of the found hole.
+	 */
+	DRM_MM_INSERT_EVICT,
+};
 
 /**
  * struct drm_mm_node - allocated block in the DRM allocator
@@ -84,14 +127,16 @@ struct drm_mm_node {
 	/** @size: Size of the allocated block. */
 	u64 size;
 	/* private: */
+	struct drm_mm *mm;
 	struct list_head node_list;
 	struct list_head hole_stack;
 	struct rb_node rb;
-	unsigned hole_follows : 1;
-	unsigned allocated : 1;
-	bool scanned_block : 1;
+	struct rb_node rb_hole_size;
+	struct rb_node rb_hole_addr;
 	u64 __subtree_last;
-	struct drm_mm *mm;
+	u64 hole_size;
+	bool allocated : 1;
+	bool scanned_block : 1;
 #ifdef CONFIG_DRM_DEBUG_MM
 	depot_stack_handle_t stack;
 #endif
@@ -127,6 +172,8 @@ struct drm_mm {
 	struct drm_mm_node head_node;
 	/* Keep an interval_tree for fast lookup of drm_mm_nodes by address. */
 	struct rb_root interval_tree;
+	struct rb_root holes_size;
+	struct rb_root holes_addr;
 
 	unsigned long scan_active;
 };
@@ -155,7 +202,7 @@ struct drm_mm_scan {
 	u64 hit_end;
 
 	unsigned long color;
-	unsigned int flags;
+	enum drm_mm_insert_mode mode;
 };
 
 /**
@@ -208,7 +255,7 @@ static inline bool drm_mm_initialized(const struct drm_mm *mm)
  */
 static inline bool drm_mm_hole_follows(const struct drm_mm_node *node)
 {
-	return node->hole_follows;
+	return node->hole_size;
 }
 
 static inline u64 __drm_mm_hole_node_start(const struct drm_mm_node *hole_node)
@@ -291,17 +338,9 @@ static inline u64 drm_mm_hole_node_end(const struct drm_mm_node *hole_node)
 #define drm_mm_for_each_node_safe(entry, next, mm) \
 	list_for_each_entry_safe(entry, next, drm_mm_nodes(mm), node_list)
 
-#define __drm_mm_for_each_hole(entry, mm, hole_start, hole_end, backwards) \
-	for (entry = list_entry((backwards) ? (mm)->hole_stack.prev : (mm)->hole_stack.next, struct drm_mm_node, hole_stack); \
-	     &entry->hole_stack != &(mm)->hole_stack ? \
-	     hole_start = drm_mm_hole_node_start(entry), \
-	     hole_end = drm_mm_hole_node_end(entry), \
-	     1 : 0; \
-	     entry = list_entry((backwards) ? entry->hole_stack.prev : entry->hole_stack.next, struct drm_mm_node, hole_stack))
-
 /**
  * drm_mm_for_each_hole - iterator to walk over all holes
- * @entry: &drm_mm_node used internally to track progress
+ * @pos: &drm_mm_node used internally to track progress
  * @mm: &drm_mm allocator to walk
  * @hole_start: ulong variable to assign the hole start to on each iteration
  * @hole_end: ulong variable to assign the hole end to on each iteration
@@ -314,57 +353,28 @@ static inline u64 drm_mm_hole_node_end(const struct drm_mm_node *hole_node)
  * Implementation Note:
  * We need to inline list_for_each_entry in order to be able to set hole_start
  * and hole_end on each iteration while keeping the macro sane.
- *
- * The __drm_mm_for_each_hole version is similar, but with added support for
- * going backwards.
  */
-#define drm_mm_for_each_hole(entry, mm, hole_start, hole_end) \
-	__drm_mm_for_each_hole(entry, mm, hole_start, hole_end, 0)
+#define drm_mm_for_each_hole(pos, mm, hole_start, hole_end) \
+	for (pos = list_first_entry(&(mm)->hole_stack, \
+				    typeof(*pos), hole_stack); \
+	     &pos->hole_stack != &(mm)->hole_stack ? \
+	     hole_start = drm_mm_hole_node_start(pos), \
+	     hole_end = hole_start + pos->hole_size, \
+	     1 : 0; \
+	     pos = list_next_entry(pos, hole_stack))
 
 /*
  * Basic range manager support (drm_mm.c)
  */
 int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node);
-int drm_mm_insert_node_in_range_generic(struct drm_mm *mm,
-					struct drm_mm_node *node,
-					u64 size,
-					u64 alignment,
-					unsigned long color,
-					u64 start,
-					u64 end,
-					enum drm_mm_search_flags sflags,
-					enum drm_mm_allocator_flags aflags);
-
-/**
- * drm_mm_insert_node_in_range - ranged search for space and insert @node
- * @mm: drm_mm to allocate from
- * @node: preallocate node to insert
- * @size: size of the allocation
- * @alignment: alignment of the allocation
- * @start: start of the allowed range for this node
- * @end: end of the allowed range for this node
- * @flags: flags to fine-tune the allocation
- *
- * This is a simplified version of drm_mm_insert_node_in_range_generic() with
- * @color set to 0.
- *
- * The preallocated node must be cleared to 0.
- *
- * Returns:
- * 0 on success, -ENOSPC if there's no suitable hole.
- */
-static inline int drm_mm_insert_node_in_range(struct drm_mm *mm,
-					      struct drm_mm_node *node,
-					      u64 size,
-					      u64 alignment,
-					      u64 start,
-					      u64 end,
-					      enum drm_mm_search_flags flags)
-{
-	return drm_mm_insert_node_in_range_generic(mm, node, size, alignment,
-						   0, start, end, flags,
-						   DRM_MM_CREATE_DEFAULT);
-}
+int drm_mm_insert_node_in_range(struct drm_mm *mm,
+				struct drm_mm_node *node,
+				u64 size,
+				u64 alignment,
+				unsigned long color,
+				u64 start,
+				u64 end,
+				enum drm_mm_insert_mode mode);
 
 /**
  * drm_mm_insert_node_generic - search for space and insert @node
@@ -373,8 +383,7 @@ static inline int drm_mm_insert_node_in_range(struct drm_mm *mm,
  * @size: size of the allocation
  * @alignment: alignment of the allocation
  * @color: opaque tag value to use for this node
- * @sflags: flags to fine-tune the allocation search
- * @aflags: flags to fine-tune the allocation behavior
+ * @mode: fine-tune the allocation search and placement
  *
  * This is a simplified version of drm_mm_insert_node_in_range_generic() with no
  * range restrictions applied.
@@ -388,13 +397,11 @@ static inline int
 drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
 			   u64 size, u64 alignment,
 			   unsigned long color,
-			   enum drm_mm_search_flags sflags,
-			   enum drm_mm_allocator_flags aflags)
+			   enum drm_mm_insert_mode mode)
 {
-	return drm_mm_insert_node_in_range_generic(mm, node,
-						   size, alignment, 0,
-						   0, U64_MAX,
-						   sflags, aflags);
+	return drm_mm_insert_node_in_range(mm, node,
+					   size, alignment, color,
+					   0, U64_MAX, mode);
 }
 
 /**
@@ -402,8 +409,6 @@ drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
  * @mm: drm_mm to allocate from
  * @node: preallocate node to insert
  * @size: size of the allocation
- * @alignment: alignment of the allocation
- * @flags: flags to fine-tune the allocation
  *
  * This is a simplified version of drm_mm_insert_node_generic() with @color set
  * to 0.
@@ -415,13 +420,9 @@ drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
  */
 static inline int drm_mm_insert_node(struct drm_mm *mm,
 				     struct drm_mm_node *node,
-				     u64 size,
-				     u64 alignment,
-				     enum drm_mm_search_flags flags)
+				     u64 size)
 {
-	return drm_mm_insert_node_generic(mm, node,
-					  size, alignment, 0,
-					  flags, DRM_MM_CREATE_DEFAULT);
+	return drm_mm_insert_node_generic(mm, node, size, 0, 0, 0);
 }
 
 void drm_mm_remove_node(struct drm_mm_node *node);
@@ -468,7 +469,7 @@ void drm_mm_scan_init_with_range(struct drm_mm_scan *scan,
 				 struct drm_mm *mm,
 				 u64 size, u64 alignment, unsigned long color,
 				 u64 start, u64 end,
-				 unsigned int flags);
+				 enum drm_mm_insert_mode mode);
 
 /**
  * drm_mm_scan_init - initialize lru scanning
@@ -477,7 +478,7 @@ void drm_mm_scan_init_with_range(struct drm_mm_scan *scan,
  * @size: size of the allocation
  * @alignment: alignment of the allocation
  * @color: opaque tag value to use for the allocation
- * @flags: flags to specify how the allocation will be performed afterwards
+ * @mode: fine-tune the allocation search and placement
  *
  * This is a simplified version of drm_mm_scan_init_with_range() with no range
  * restrictions applied.
@@ -494,12 +495,11 @@ static inline void drm_mm_scan_init(struct drm_mm_scan *scan,
 				    u64 size,
 				    u64 alignment,
 				    unsigned long color,
-				    unsigned int flags)
+				    enum drm_mm_insert_mode mode)
 {
 	drm_mm_scan_init_with_range(scan, mm,
 				    size, alignment, color,
-				    0, U64_MAX,
-				    flags);
+				    0, U64_MAX, mode);
 }
 
 bool drm_mm_scan_add_block(struct drm_mm_scan *scan,