From a7f014f2de04893f95cfe40fe35f15c8dae4b36e Mon Sep 17 00:00:00 2001
From: Eric Anholt <eric@anholt.net>
Date: Tue, 25 Nov 2008 14:02:05 -0800
Subject: drm/i915: Respect GM965/GM45 bit-17-instead-of-bit-11 option for
 swizzling.

This fixes readpixels and buffer corruption when swapped out and in by
disabling tiling on them.

Now that we know that the bit 17 mode isn't just a mistake of older chipsets,
we'll need to work on a clever fix so that we can get the performance of
tiling on these chipsets, but that will require intrusive changes targeted
at the next kernel release, not this one.

Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_gem_tiling.c | 7 ++++---
 drivers/gpu/drm/i915/i915_reg.h        | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index e8b85ac4ca04..a8cb69469c64 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -119,9 +119,10 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 			    dcc & DCC_CHANNEL_XOR_DISABLE) {
 				swizzle_x = I915_BIT_6_SWIZZLE_9_10;
 				swizzle_y = I915_BIT_6_SWIZZLE_9;
-			} else if (IS_I965GM(dev) || IS_GM45(dev)) {
-				/* GM965 only does bit 11-based channel
-				 * randomization
+			} else if ((IS_I965GM(dev) || IS_GM45(dev)) &&
+				   (dcc & DCC_CHANNEL_XOR_BIT_17) == 0) {
+				/* GM965/GM45 does either bit 11 or bit 17
+				 * swizzling.
 				 */
 				swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
 				swizzle_y = I915_BIT_6_SWIZZLE_9_11;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 0e476eba36e6..9d24aaeb8a45 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -522,6 +522,7 @@
 #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED	(2 << 0)
 #define DCC_ADDRESSING_MODE_MASK			(3 << 0)
 #define DCC_CHANNEL_XOR_DISABLE				(1 << 10)
+#define DCC_CHANNEL_XOR_BIT_17				(1 << 9)
 
 /** 965 MCH register controlling DRAM channel configuration */
 #define C0DRB3			0x10206
-- 
cgit v1.2.1


From ce44b0ea3dc04236d852d78a06f850d1f7b03f3d Mon Sep 17 00:00:00 2001
From: Eric Anholt <eric@anholt.net>
Date: Thu, 6 Nov 2008 16:00:31 -0800
Subject: drm/i915: Move flushing list cleanup from flush request retire to
 request emit.

obj_priv->write_domain is "write domain if the GPU went idle now", not
"write domain at this moment."  By postponing the clear, we confused the
concept, required more storage, and potentially emitted more flushes than
are required.

Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_drv.h      | 11 ++++--
 drivers/gpu/drm/i915/i915_gem.c      | 73 ++++++++++++++++++++----------------
 drivers/gpu/drm/i915/i915_gem_proc.c |  5 +--
 3 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 0a4f39b9a0ec..76d9a706d8fd 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -244,6 +244,10 @@ typedef struct drm_i915_private {
 		 * List of objects currently involved in rendering from the
 		 * ringbuffer.
 		 *
+		 * Includes buffers having the contents of their GPU caches
+		 * flushed, not necessarily primitives.  last_rendering_seqno
+		 * represents when the rendering involved will be completed.
+		 *
 		 * A reference is held on the buffer while on this list.
 		 */
 		struct list_head active_list;
@@ -253,6 +257,8 @@ typedef struct drm_i915_private {
 		 * still have a write_domain which needs to be flushed before
 		 * unbinding.
 		 *
+		 * last_rendering_seqno is 0 while an object is in this list.
+		 *
 		 * A reference is held on the buffer while on this list.
 		 */
 		struct list_head flushing_list;
@@ -261,6 +267,8 @@ typedef struct drm_i915_private {
 		 * LRU list of objects which are not in the ringbuffer and
 		 * are ready to unbind, but are still in the GTT.
 		 *
+		 * last_rendering_seqno is 0 while an object is in this list.
+		 *
 		 * A reference is not held on the buffer while on this list,
 		 * as merely being GTT-bound shouldn't prevent its being
 		 * freed, and we'll pull it off the list in the free path.
@@ -394,9 +402,6 @@ struct drm_i915_gem_request {
 	/** Time at which this request was emitted, in jiffies. */
 	unsigned long emitted_jiffies;
 
-	/** Cache domains that were flushed at the start of the request. */
-	uint32_t flush_domains;
-
 	struct list_head list;
 };
 
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index d58ddef468f8..9fd28ebe0aa3 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -532,7 +532,7 @@ i915_gem_object_free_page_list(struct drm_gem_object *obj)
 }
 
 static void
-i915_gem_object_move_to_active(struct drm_gem_object *obj)
+i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
 {
 	struct drm_device *dev = obj->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
@@ -546,8 +546,20 @@ i915_gem_object_move_to_active(struct drm_gem_object *obj)
 	/* Move from whatever list we were on to the tail of execution. */
 	list_move_tail(&obj_priv->list,
 		       &dev_priv->mm.active_list);
+	obj_priv->last_rendering_seqno = seqno;
 }
 
+static void
+i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+
+	BUG_ON(!obj_priv->active);
+	list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
+	obj_priv->last_rendering_seqno = 0;
+}
 
 static void
 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
@@ -562,6 +574,7 @@ i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
 	else
 		list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
 
+	obj_priv->last_rendering_seqno = 0;
 	if (obj_priv->active) {
 		obj_priv->active = 0;
 		drm_gem_object_unreference(obj);
@@ -610,10 +623,28 @@ i915_add_request(struct drm_device *dev, uint32_t flush_domains)
 
 	request->seqno = seqno;
 	request->emitted_jiffies = jiffies;
-	request->flush_domains = flush_domains;
 	was_empty = list_empty(&dev_priv->mm.request_list);
 	list_add_tail(&request->list, &dev_priv->mm.request_list);
 
+	/* Associate any objects on the flushing list matching the write
+	 * domain we're flushing with our flush.
+	 */
+	if (flush_domains != 0) {
+		struct drm_i915_gem_object *obj_priv, *next;
+
+		list_for_each_entry_safe(obj_priv, next,
+					 &dev_priv->mm.flushing_list, list) {
+			struct drm_gem_object *obj = obj_priv->obj;
+
+			if ((obj->write_domain & flush_domains) ==
+			    obj->write_domain) {
+				obj->write_domain = 0;
+				i915_gem_object_move_to_active(obj, seqno);
+			}
+		}
+
+	}
+
 	if (was_empty && !dev_priv->mm.suspended)
 		schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
 	return seqno;
@@ -676,30 +707,10 @@ i915_gem_retire_request(struct drm_device *dev,
 			 __func__, request->seqno, obj);
 #endif
 
-		if (obj->write_domain != 0) {
-			list_move_tail(&obj_priv->list,
-				       &dev_priv->mm.flushing_list);
-		} else {
+		if (obj->write_domain != 0)
+			i915_gem_object_move_to_flushing(obj);
+		else
 			i915_gem_object_move_to_inactive(obj);
-		}
-	}
-
-	if (request->flush_domains != 0) {
-		struct drm_i915_gem_object *obj_priv, *next;
-
-		/* Clear the write domain and activity from any buffers
-		 * that are just waiting for a flush matching the one retired.
-		 */
-		list_for_each_entry_safe(obj_priv, next,
-					 &dev_priv->mm.flushing_list, list) {
-			struct drm_gem_object *obj = obj_priv->obj;
-
-			if (obj->write_domain & request->flush_domains) {
-				obj->write_domain = 0;
-				i915_gem_object_move_to_inactive(obj);
-			}
-		}
-
 	}
 }
 
@@ -896,17 +907,15 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj)
 	 * create a new seqno to wait for.
 	 */
 	if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) {
-		uint32_t write_domain = obj->write_domain;
+		uint32_t seqno, write_domain = obj->write_domain;
 #if WATCH_BUF
 		DRM_INFO("%s: flushing object %p from write domain %08x\n",
 			  __func__, obj, write_domain);
 #endif
 		i915_gem_flush(dev, 0, write_domain);
 
-		i915_gem_object_move_to_active(obj);
-		obj_priv->last_rendering_seqno = i915_add_request(dev,
-								  write_domain);
-		BUG_ON(obj_priv->last_rendering_seqno == 0);
+		seqno = i915_add_request(dev, write_domain);
+		i915_gem_object_move_to_active(obj, seqno);
 #if WATCH_LRU
 		DRM_INFO("%s: flush moves to exec list %p\n", __func__, obj);
 #endif
@@ -1927,10 +1936,8 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 	i915_file_priv->mm.last_gem_seqno = seqno;
 	for (i = 0; i < args->buffer_count; i++) {
 		struct drm_gem_object *obj = object_list[i];
-		struct drm_i915_gem_object *obj_priv = obj->driver_private;
 
-		i915_gem_object_move_to_active(obj);
-		obj_priv->last_rendering_seqno = seqno;
+		i915_gem_object_move_to_active(obj, seqno);
 #if WATCH_LRU
 		DRM_INFO("%s: move to exec list %p\n", __func__, obj);
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem_proc.c b/drivers/gpu/drm/i915/i915_gem_proc.c
index 93de15b4c9a7..e8d5abe1250e 100644
--- a/drivers/gpu/drm/i915/i915_gem_proc.c
+++ b/drivers/gpu/drm/i915/i915_gem_proc.c
@@ -166,10 +166,9 @@ static int i915_gem_request_info(char *buf, char **start, off_t offset,
 	list_for_each_entry(gem_request, &dev_priv->mm.request_list,
 			    list)
 	{
-		DRM_PROC_PRINT("    %d @ %d %08x\n",
+		DRM_PROC_PRINT("    %d @ %d\n",
 			       gem_request->seqno,
-			       (int) (jiffies - gem_request->emitted_jiffies),
-			       gem_request->flush_domains);
+			       (int) (jiffies - gem_request->emitted_jiffies));
 	}
 	if (len > request + offset)
 		return request;
-- 
cgit v1.2.1


From b670d8158283c35842ae1c650f75c375d8710607 Mon Sep 17 00:00:00 2001
From: Eric Anholt <eric@anholt.net>
Date: Fri, 14 Nov 2008 16:27:47 -0800
Subject: drm/i915: If interrupted while setting object domains, still emit the
 flush.

Otherwise, we would leave the objects in an inconsistent state, such as
write_domain == 0 but on the flushing list.

Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_gem.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 9fd28ebe0aa3..3fe108b7e2fa 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1880,8 +1880,20 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 		ret = i915_gem_object_set_domain(obj,
 						 obj->pending_read_domains,
 						 obj->pending_write_domain);
-		if (ret)
+		if (ret) {
+			/* As we've partially updated domains on our buffers,
+			 * we have to emit the flush we've accumulated
+			 * before exiting, or we'll have broken the
+			 * active/flushing/inactive invariants.
+			 *
+			 * We'll potentially have some things marked as
+			 * being in write domains that they actually aren't,
+			 * but that should be merely a minor performance loss.
+			 */
+			flush_domains = i915_gem_dev_set_domain(dev);
+			(void)i915_add_request(dev, flush_domains);
 			goto err;
+		}
 	}
 
 	i915_verify_inactive(dev, __FILE__, __LINE__);
-- 
cgit v1.2.1


From 2ef7eeaa553d88e78d9a4520271f26a7bc0e2968 Mon Sep 17 00:00:00 2001
From: Eric Anholt <eric@anholt.net>
Date: Mon, 10 Nov 2008 10:53:25 -0800
Subject: drm/i915: Make a single set-to-gtt-domain path.

This fixes failure to flush caches in the relocation update path, and
failure to wait in the set_domain ioctl, each of which could lead to incorrect
rendering.

Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_gem.c | 115 +++++++++++++++++++++++++++++++++-------
 1 file changed, 96 insertions(+), 19 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 3fe108b7e2fa..62a059dce85f 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -48,6 +48,8 @@ i915_gem_set_domain(struct drm_gem_object *obj,
 		    struct drm_file *file_priv,
 		    uint32_t read_domains,
 		    uint32_t write_domain);
+static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
+					     int write);
 static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
 static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
@@ -260,8 +262,7 @@ i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
 		mutex_unlock(&dev->struct_mutex);
 		return ret;
 	}
-	ret = i915_gem_set_domain(obj, file_priv,
-				  I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
+	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 	if (ret)
 		goto fail;
 
@@ -397,7 +398,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 }
 
 /**
- * Called when user space prepares to use an object
+ * Called when user space prepares to use an object with the CPU, either
+ * through the mmap ioctl's mapping or a GTT mapping.
  */
 int
 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
@@ -405,11 +407,26 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_set_domain *args = data;
 	struct drm_gem_object *obj;
+	uint32_t read_domains = args->read_domains;
+	uint32_t write_domain = args->write_domain;
 	int ret;
 
 	if (!(dev->driver->driver_features & DRIVER_GEM))
 		return -ENODEV;
 
+	/* Only handle setting domains to types used by the CPU. */
+	if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+		return -EINVAL;
+
+	if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+		return -EINVAL;
+
+	/* Having something in the write domain implies it's in the read
+	 * domain, and only that read domain.  Enforce that in the request.
+	 */
+	if (write_domain != 0 && read_domains != write_domain)
+		return -EINVAL;
+
 	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 	if (obj == NULL)
 		return -EBADF;
@@ -417,10 +434,15 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	mutex_lock(&dev->struct_mutex);
 #if WATCH_BUF
 	DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
-		 obj, obj->size, args->read_domains, args->write_domain);
+		 obj, obj->size, read_domains, write_domain);
 #endif
-	ret = i915_gem_set_domain(obj, file_priv,
-				  args->read_domains, args->write_domain);
+	if (read_domains & I915_GEM_DOMAIN_GTT) {
+		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
+	} else {
+		ret = i915_gem_set_domain(obj, file_priv,
+					  read_domains, write_domain);
+	}
+
 	drm_gem_object_unreference(obj);
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
@@ -1237,6 +1259,69 @@ i915_gem_clflush_object(struct drm_gem_object *obj)
 	drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
 }
 
+/**
+ * Moves a single object to the GTT read, and possibly write domain.
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+static int
+i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
+{
+	struct drm_device *dev = obj->dev;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	uint32_t flush_domains;
+
+	/* Figure out what GPU domains we need to flush or invalidate for
+	 * moving to GTT.
+	 */
+	flush_domains = obj->write_domain & I915_GEM_GPU_DOMAINS;
+
+	/* Queue the GPU write cache flushing we need. */
+	if (flush_domains != 0) {
+		uint32_t seqno;
+
+		obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
+		i915_gem_flush(dev, 0, flush_domains);
+		seqno = i915_add_request(dev, flush_domains);
+		i915_gem_object_move_to_active(obj, seqno);
+	}
+
+	/* Wait on any GPU rendering and flushing to occur. */
+	if (obj_priv->active) {
+		int ret;
+
+		ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
+		if (ret != 0)
+			return ret;
+	}
+
+	/* If we're writing through the GTT domain, then CPU and GPU caches
+	 * will need to be invalidated at next use.
+	 */
+	if (write)
+		obj->read_domains &= ~(I915_GEM_GPU_DOMAINS |
+					    I915_GEM_DOMAIN_CPU);
+
+	/* Flush the CPU domain if it's dirty. */
+	if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
+		i915_gem_clflush_object(obj);
+		drm_agp_chipset_flush(dev);
+
+		obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
+	}
+
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
+	obj->read_domains |= I915_GEM_DOMAIN_GTT;
+	if (write)
+		obj->write_domain = I915_GEM_DOMAIN_GTT;
+
+	return 0;
+}
+
 /*
  * Set the next domain for the specified object. This
  * may not actually perform the necessary flushing/invaliding though,
@@ -1634,19 +1719,11 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
 			continue;
 		}
 
-		/* Now that we're going to actually write some data in,
-		 * make sure that any rendering using this buffer's contents
-		 * is completed.
-		 */
-		i915_gem_object_wait_rendering(obj);
-
-		/* As we're writing through the gtt, flush
-		 * any CPU writes before we write the relocations
-		 */
-		if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
-			i915_gem_clflush_object(obj);
-			drm_agp_chipset_flush(dev);
-			obj->write_domain = 0;
+		ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+		if (ret != 0) {
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			return -EINVAL;
 		}
 
 		/* Map the page containing the relocation we're going to
-- 
cgit v1.2.1


From e47c68e9c5d71e2faab8c2b82f57c6c73e6456fd Mon Sep 17 00:00:00 2001
From: Eric Anholt <eric@anholt.net>
Date: Fri, 14 Nov 2008 13:35:19 -0800
Subject: drm/i915: Make a single set-to-cpu-domain path and use it wherever
 needed.

This fixes several domain management bugs, including potential lack of cache
invalidation for pread, potential failure to wait for set_domain(CPU, 0),
and more, along with producing more intelligible code.

Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_drv.h |   4 +-
 drivers/gpu/drm/i915/i915_gem.c | 363 +++++++++++++++++++++++-----------------
 2 files changed, 215 insertions(+), 152 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 76d9a706d8fd..adc972cc6bfc 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -379,8 +379,8 @@ struct drm_i915_gem_object {
 	uint32_t agp_type;
 
 	/**
-	 * Flagging of which individual pages are valid in GEM_DOMAIN_CPU when
-	 * GEM_DOMAIN_CPU is not in the object's read domain.
+	 * If present, while GEM_DOMAIN_CPU is in the read domain this array
+	 * flags which individual pages are valid.
 	 */
 	uint8_t *page_cpu_valid;
 };
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 62a059dce85f..f7e9f2c2934c 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -37,19 +37,17 @@ static int
 i915_gem_object_set_domain(struct drm_gem_object *obj,
 			    uint32_t read_domains,
 			    uint32_t write_domain);
-static int
-i915_gem_object_set_domain_range(struct drm_gem_object *obj,
-				 uint64_t offset,
-				 uint64_t size,
-				 uint32_t read_domains,
-				 uint32_t write_domain);
-static int
-i915_gem_set_domain(struct drm_gem_object *obj,
-		    struct drm_file *file_priv,
-		    uint32_t read_domains,
-		    uint32_t write_domain);
+static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
+static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
+static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
 static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
 					     int write);
+static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
+					     int write);
+static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
+						     uint64_t offset,
+						     uint64_t size);
+static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
 static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
 static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
@@ -164,8 +162,8 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 
 	mutex_lock(&dev->struct_mutex);
 
-	ret = i915_gem_object_set_domain_range(obj, args->offset, args->size,
-					       I915_GEM_DOMAIN_CPU, 0);
+	ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
+							args->size);
 	if (ret != 0) {
 		drm_gem_object_unreference(obj);
 		mutex_unlock(&dev->struct_mutex);
@@ -321,8 +319,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
 
 	mutex_lock(&dev->struct_mutex);
 
-	ret = i915_gem_set_domain(obj, file_priv,
-				  I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
+	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 	if (ret) {
 		mutex_unlock(&dev->struct_mutex);
 		return ret;
@@ -439,8 +436,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	if (read_domains & I915_GEM_DOMAIN_GTT) {
 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 	} else {
-		ret = i915_gem_set_domain(obj, file_priv,
-					  read_domains, write_domain);
+		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 	}
 
 	drm_gem_object_unreference(obj);
@@ -477,10 +473,9 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 	obj_priv = obj->driver_private;
 
 	/* Pinned buffers may be scanout, so flush the cache */
-	if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) {
-		i915_gem_clflush_object(obj);
-		drm_agp_chipset_flush(dev);
-	}
+	if (obj_priv->pin_count)
+		i915_gem_object_flush_cpu_write_domain(obj);
+
 	drm_gem_object_unreference(obj);
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
@@ -925,23 +920,10 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj)
 	struct drm_i915_gem_object *obj_priv = obj->driver_private;
 	int ret;
 
-	/* If there are writes queued to the buffer, flush and
-	 * create a new seqno to wait for.
+	/* This function only exists to support waiting for existing rendering,
+	 * not for emitting required flushes.
 	 */
-	if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) {
-		uint32_t seqno, write_domain = obj->write_domain;
-#if WATCH_BUF
-		DRM_INFO("%s: flushing object %p from write domain %08x\n",
-			  __func__, obj, write_domain);
-#endif
-		i915_gem_flush(dev, 0, write_domain);
-
-		seqno = i915_add_request(dev, write_domain);
-		i915_gem_object_move_to_active(obj, seqno);
-#if WATCH_LRU
-		DRM_INFO("%s: flush moves to exec list %p\n", __func__, obj);
-#endif
-	}
+	BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
 
 	/* If there is rendering queued on the buffer being evicted, wait for
 	 * it.
@@ -981,24 +963,16 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
 		return -EINVAL;
 	}
 
-	/* Wait for any rendering to complete
-	 */
-	ret = i915_gem_object_wait_rendering(obj);
-	if (ret) {
-		DRM_ERROR("wait_rendering failed: %d\n", ret);
-		return ret;
-	}
-
 	/* Move the object to the CPU domain to ensure that
 	 * any possible CPU writes while it's not in the GTT
 	 * are flushed when we go to remap it. This will
 	 * also ensure that all pending GPU writes are finished
 	 * before we unbind.
 	 */
-	ret = i915_gem_object_set_domain(obj, I915_GEM_DOMAIN_CPU,
-					 I915_GEM_DOMAIN_CPU);
+	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 	if (ret) {
-		DRM_ERROR("set_domain failed: %d\n", ret);
+		if (ret != -ERESTARTSYS)
+			DRM_ERROR("set_domain failed: %d\n", ret);
 		return ret;
 	}
 
@@ -1259,6 +1233,51 @@ i915_gem_clflush_object(struct drm_gem_object *obj)
 	drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
 }
 
+/** Flushes any GPU write domain for the object if it's dirty. */
+static void
+i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	uint32_t seqno;
+
+	if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
+		return;
+
+	/* Queue the GPU write cache flushing we need. */
+	i915_gem_flush(dev, 0, obj->write_domain);
+	seqno = i915_add_request(dev, obj->write_domain);
+	obj->write_domain = 0;
+	i915_gem_object_move_to_active(obj, seqno);
+}
+
+/** Flushes the GTT write domain for the object if it's dirty. */
+static void
+i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
+{
+	if (obj->write_domain != I915_GEM_DOMAIN_GTT)
+		return;
+
+	/* No actual flushing is required for the GTT write domain.   Writes
+	 * to it immediately go to main memory as far as we know, so there's
+	 * no chipset flush.  It also doesn't land in render cache.
+	 */
+	obj->write_domain = 0;
+}
+
+/** Flushes the CPU write domain for the object if it's dirty. */
+static void
+i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+
+	if (obj->write_domain != I915_GEM_DOMAIN_CPU)
+		return;
+
+	i915_gem_clflush_object(obj);
+	drm_agp_chipset_flush(dev);
+	obj->write_domain = 0;
+}
+
 /**
  * Moves a single object to the GTT read, and possibly write domain.
  *
@@ -1268,56 +1287,81 @@ i915_gem_clflush_object(struct drm_gem_object *obj)
 static int
 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
 {
-	struct drm_device *dev = obj->dev;
 	struct drm_i915_gem_object *obj_priv = obj->driver_private;
-	uint32_t flush_domains;
+	int ret;
 
-	/* Figure out what GPU domains we need to flush or invalidate for
-	 * moving to GTT.
+	i915_gem_object_flush_gpu_write_domain(obj);
+	/* Wait on any GPU rendering and flushing to occur. */
+	ret = i915_gem_object_wait_rendering(obj);
+	if (ret != 0)
+		return ret;
+
+	/* If we're writing through the GTT domain, then CPU and GPU caches
+	 * will need to be invalidated at next use.
 	 */
-	flush_domains = obj->write_domain & I915_GEM_GPU_DOMAINS;
+	if (write)
+		obj->read_domains &= I915_GEM_DOMAIN_GTT;
 
-	/* Queue the GPU write cache flushing we need. */
-	if (flush_domains != 0) {
-		uint32_t seqno;
+	i915_gem_object_flush_cpu_write_domain(obj);
 
-		obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
-		i915_gem_flush(dev, 0, flush_domains);
-		seqno = i915_add_request(dev, flush_domains);
-		i915_gem_object_move_to_active(obj, seqno);
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
+	obj->read_domains |= I915_GEM_DOMAIN_GTT;
+	if (write) {
+		obj->write_domain = I915_GEM_DOMAIN_GTT;
+		obj_priv->dirty = 1;
 	}
 
+	return 0;
+}
+
+/**
+ * Moves a single object to the CPU read, and possibly write domain.
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+static int
+i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
+{
+	struct drm_device *dev = obj->dev;
+	int ret;
+
+	i915_gem_object_flush_gpu_write_domain(obj);
 	/* Wait on any GPU rendering and flushing to occur. */
-	if (obj_priv->active) {
-		int ret;
+	ret = i915_gem_object_wait_rendering(obj);
+	if (ret != 0)
+		return ret;
 
-		ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
-		if (ret != 0)
-			return ret;
-	}
+	i915_gem_object_flush_gtt_write_domain(obj);
 
-	/* If we're writing through the GTT domain, then CPU and GPU caches
-	 * will need to be invalidated at next use.
+	/* If we have a partially-valid cache of the object in the CPU,
+	 * finish invalidating it and free the per-page flags.
 	 */
-	if (write)
-		obj->read_domains &= ~(I915_GEM_GPU_DOMAINS |
-					    I915_GEM_DOMAIN_CPU);
+	i915_gem_object_set_to_full_cpu_read_domain(obj);
 
-	/* Flush the CPU domain if it's dirty. */
-	if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
+	/* Flush the CPU cache if it's still invalid. */
+	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
 		i915_gem_clflush_object(obj);
 		drm_agp_chipset_flush(dev);
 
-		obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
+		obj->read_domains |= I915_GEM_DOMAIN_CPU;
 	}
 
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
 	 */
-	BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
-	obj->read_domains |= I915_GEM_DOMAIN_GTT;
-	if (write)
-		obj->write_domain = I915_GEM_DOMAIN_GTT;
+	BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+
+	/* If we're writing through the CPU, then the GPU read domains will
+	 * need to be invalidated at next use.
+	 */
+	if (write) {
+		obj->read_domains &= I915_GEM_DOMAIN_CPU;
+		obj->write_domain = I915_GEM_DOMAIN_CPU;
+	}
 
 	return 0;
 }
@@ -1442,7 +1486,9 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
 	struct drm_i915_gem_object	*obj_priv = obj->driver_private;
 	uint32_t			invalidate_domains = 0;
 	uint32_t			flush_domains = 0;
-	int				ret;
+
+	BUG_ON(read_domains & I915_GEM_DOMAIN_CPU);
+	BUG_ON(write_domain == I915_GEM_DOMAIN_CPU);
 
 #if WATCH_BUF
 	DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
@@ -1479,34 +1525,11 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
 		DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
 			 __func__, flush_domains, invalidate_domains);
 #endif
-		/*
-		 * If we're invaliding the CPU cache and flushing a GPU cache,
-		 * then pause for rendering so that the GPU caches will be
-		 * flushed before the cpu cache is invalidated
-		 */
-		if ((invalidate_domains & I915_GEM_DOMAIN_CPU) &&
-		    (flush_domains & ~(I915_GEM_DOMAIN_CPU |
-				       I915_GEM_DOMAIN_GTT))) {
-			ret = i915_gem_object_wait_rendering(obj);
-			if (ret)
-				return ret;
-		}
 		i915_gem_clflush_object(obj);
 	}
 
 	if ((write_domain | flush_domains) != 0)
 		obj->write_domain = write_domain;
-
-	/* If we're invalidating the CPU domain, clear the per-page CPU
-	 * domain list as well.
-	 */
-	if (obj_priv->page_cpu_valid != NULL &&
-	    (write_domain != 0 ||
-	     read_domains & I915_GEM_DOMAIN_CPU)) {
-		drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
-			 DRM_MEM_DRIVER);
-		obj_priv->page_cpu_valid = NULL;
-	}
 	obj->read_domains = read_domains;
 
 	dev->invalidate_domains |= invalidate_domains;
@@ -1521,43 +1544,91 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
 }
 
 /**
- * Set the read/write domain on a range of the object.
+ * Moves the object from a partially CPU read to a full one.
  *
- * Currently only implemented for CPU reads, otherwise drops to normal
- * i915_gem_object_set_domain().
+ * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
+ * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  */
-static int
-i915_gem_object_set_domain_range(struct drm_gem_object *obj,
-				 uint64_t offset,
-				 uint64_t size,
-				 uint32_t read_domains,
-				 uint32_t write_domain)
+static void
+i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
 {
+	struct drm_device *dev = obj->dev;
 	struct drm_i915_gem_object *obj_priv = obj->driver_private;
-	int ret, i;
 
-	if (obj->read_domains & I915_GEM_DOMAIN_CPU)
-		return 0;
+	if (!obj_priv->page_cpu_valid)
+		return;
 
-	if (read_domains != I915_GEM_DOMAIN_CPU ||
-	    write_domain != 0)
-		return i915_gem_object_set_domain(obj,
-						  read_domains, write_domain);
+	/* If we're partially in the CPU read domain, finish moving it in.
+	 */
+	if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
+		int i;
+
+		for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
+			if (obj_priv->page_cpu_valid[i])
+				continue;
+			drm_clflush_pages(obj_priv->page_list + i, 1);
+		}
+		drm_agp_chipset_flush(dev);
+	}
 
-	/* Wait on any GPU rendering to the object to be flushed. */
+	/* Free the page_cpu_valid mappings which are now stale, whether
+	 * or not we've got I915_GEM_DOMAIN_CPU.
+	 */
+	drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
+		 DRM_MEM_DRIVER);
+	obj_priv->page_cpu_valid = NULL;
+}
+
+/**
+ * Set the CPU read domain on a range of the object.
+ *
+ * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
+ * not entirely valid.  The page_cpu_valid member of the object flags which
+ * pages have been flushed, and will be respected by
+ * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
+ * of the whole object.
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+static int
+i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
+					  uint64_t offset, uint64_t size)
+{
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	int i, ret;
+
+	if (offset == 0 && size == obj->size)
+		return i915_gem_object_set_to_cpu_domain(obj, 0);
+
+	i915_gem_object_flush_gpu_write_domain(obj);
+	/* Wait on any GPU rendering and flushing to occur. */
 	ret = i915_gem_object_wait_rendering(obj);
-	if (ret)
+	if (ret != 0)
 		return ret;
+	i915_gem_object_flush_gtt_write_domain(obj);
+
+	/* If we're already fully in the CPU read domain, we're done. */
+	if (obj_priv->page_cpu_valid == NULL &&
+	    (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
+		return 0;
 
+	/* Otherwise, create/clear the per-page CPU read domain flag if we're
+	 * newly adding I915_GEM_DOMAIN_CPU
+	 */
 	if (obj_priv->page_cpu_valid == NULL) {
 		obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
 						      DRM_MEM_DRIVER);
-	}
+		if (obj_priv->page_cpu_valid == NULL)
+			return -ENOMEM;
+	} else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
+		memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
 
 	/* Flush the cache on any pages that are still invalid from the CPU's
 	 * perspective.
 	 */
-	for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) {
+	for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
+	     i++) {
 		if (obj_priv->page_cpu_valid[i])
 			continue;
 
@@ -1566,6 +1637,13 @@ i915_gem_object_set_domain_range(struct drm_gem_object *obj,
 		obj_priv->page_cpu_valid[i] = 1;
 	}
 
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+
+	obj->read_domains |= I915_GEM_DOMAIN_CPU;
+
 	return 0;
 }
 
@@ -1679,6 +1757,18 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
 			return -EINVAL;
 		}
 
+		if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
+		    reloc.read_domains & I915_GEM_DOMAIN_CPU) {
+			DRM_ERROR("reloc with read/write CPU domains: "
+				  "obj %p target %d offset %d "
+				  "read %08x write %08x",
+				  obj, reloc.target_handle,
+				  (int) reloc.offset,
+				  reloc.read_domains,
+				  reloc.write_domain);
+			return -EINVAL;
+		}
+
 		if (reloc.write_domain && target_obj->pending_write_domain &&
 		    reloc.write_domain != target_obj->pending_write_domain) {
 			DRM_ERROR("Write domain conflict: "
@@ -2157,11 +2247,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 	/* XXX - flush the CPU caches for pinned objects
 	 * as the X server doesn't manage domains yet
 	 */
-	if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
-		i915_gem_clflush_object(obj);
-		drm_agp_chipset_flush(dev);
-		obj->write_domain = 0;
-	}
+	i915_gem_object_flush_cpu_write_domain(obj);
 	args->offset = obj_priv->gtt_offset;
 	drm_gem_object_unreference(obj);
 	mutex_unlock(&dev->struct_mutex);
@@ -2263,29 +2349,6 @@ void i915_gem_free_object(struct drm_gem_object *obj)
 	drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
 }
 
-static int
-i915_gem_set_domain(struct drm_gem_object *obj,
-		    struct drm_file *file_priv,
-		    uint32_t read_domains,
-		    uint32_t write_domain)
-{
-	struct drm_device *dev = obj->dev;
-	int ret;
-	uint32_t flush_domains;
-
-	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
-
-	ret = i915_gem_object_set_domain(obj, read_domains, write_domain);
-	if (ret)
-		return ret;
-	flush_domains = i915_gem_dev_set_domain(obj->dev);
-
-	if (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))
-		(void) i915_add_request(dev, flush_domains);
-
-	return 0;
-}
-
 /** Unbinds all objects that are on the given buffer list. */
 static int
 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
-- 
cgit v1.2.1


From c0d90829288942fa06d7483f9e84059a64605da5 Mon Sep 17 00:00:00 2001
From: Keith Packard <keithp@keithp.com>
Date: Thu, 20 Nov 2008 23:11:08 -0800
Subject: drm/i915: Rename object_set_domain to object_set_to_gpu_domain

Now that the CPU and GTT domain operations are isolated to their own
functions, the previously general-purpose set_domain function is now used
only to set GPU domains. It also has no failure cases, which is important as
this eliminates any possible interruption of the computation of new object
domains and subsequent emmission of the flushing instructions into the ring.

Signed-off-by: Keith Packard <keithp@keithp.com>
Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_gem.c | 39 ++++++++++++---------------------------
 1 file changed, 12 insertions(+), 27 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index f7e9f2c2934c..50c75327d567 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -33,10 +33,10 @@
 
 #define I915_GEM_GPU_DOMAINS	(~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
 
-static int
-i915_gem_object_set_domain(struct drm_gem_object *obj,
-			    uint32_t read_domains,
-			    uint32_t write_domain);
+static void
+i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
+				  uint32_t read_domains,
+				  uint32_t write_domain);
 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
@@ -1477,10 +1477,10 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
  *		MI_FLUSH
  *		drm_agp_chipset_flush
  */
-static int
-i915_gem_object_set_domain(struct drm_gem_object *obj,
-			    uint32_t read_domains,
-			    uint32_t write_domain)
+static void
+i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
+				  uint32_t read_domains,
+				  uint32_t write_domain)
 {
 	struct drm_device		*dev = obj->dev;
 	struct drm_i915_gem_object	*obj_priv = obj->driver_private;
@@ -1540,7 +1540,6 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
 		 obj->read_domains, obj->write_domain,
 		 dev->invalidate_domains, dev->flush_domains);
 #endif
-	return 0;
 }
 
 /**
@@ -2043,24 +2042,10 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 	for (i = 0; i < args->buffer_count; i++) {
 		struct drm_gem_object *obj = object_list[i];
 
-		/* make sure all previous memory operations have passed */
-		ret = i915_gem_object_set_domain(obj,
-						 obj->pending_read_domains,
-						 obj->pending_write_domain);
-		if (ret) {
-			/* As we've partially updated domains on our buffers,
-			 * we have to emit the flush we've accumulated
-			 * before exiting, or we'll have broken the
-			 * active/flushing/inactive invariants.
-			 *
-			 * We'll potentially have some things marked as
-			 * being in write domains that they actually aren't,
-			 * but that should be merely a minor performance loss.
-			 */
-			flush_domains = i915_gem_dev_set_domain(dev);
-			(void)i915_add_request(dev, flush_domains);
-			goto err;
-		}
+		/* Compute new gpu domains and update invalidate/flushing */
+		i915_gem_object_set_to_gpu_domain(obj,
+						  obj->pending_read_domains,
+						  obj->pending_write_domain);
 	}
 
 	i915_verify_inactive(dev, __FILE__, __LINE__);
-- 
cgit v1.2.1


From 646f0f6e43bf6628b1f0f8ca6c0227ce72e8ef3c Mon Sep 17 00:00:00 2001
From: Keith Packard <keithp@keithp.com>
Date: Thu, 20 Nov 2008 23:23:03 -0800
Subject: drm/i915: Move the execbuffer domain computations together

This eliminates the dev_set_domain function and just in-lines it
where its used, with the goal of moving the manipulation and use of
invalidate_domains and flush_domains closer together. This also
avoids calling add_request unless some domain has been flushed.

Signed-off-by: Keith Packard <keithp@keithp.com>
Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_gem.c | 65 +++++++++++++----------------------------
 1 file changed, 21 insertions(+), 44 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 50c75327d567..c171a2d93bb6 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1646,38 +1646,6 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
 	return 0;
 }
 
-/**
- * Once all of the objects have been set in the proper domain,
- * perform the necessary flush and invalidate operations.
- *
- * Returns the write domains flushed, for use in flush tracking.
- */
-static uint32_t
-i915_gem_dev_set_domain(struct drm_device *dev)
-{
-	uint32_t flush_domains = dev->flush_domains;
-
-	/*
-	 * Now that all the buffers are synced to the proper domains,
-	 * flush and invalidate the collected domains
-	 */
-	if (dev->invalidate_domains | dev->flush_domains) {
-#if WATCH_EXEC
-		DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
-			  __func__,
-			 dev->invalidate_domains,
-			 dev->flush_domains);
-#endif
-		i915_gem_flush(dev,
-			       dev->invalidate_domains,
-			       dev->flush_domains);
-		dev->invalidate_domains = 0;
-		dev->flush_domains = 0;
-	}
-
-	return flush_domains;
-}
-
 /**
  * Pin an object to the GTT and evaluate the relocations landing in it.
  */
@@ -2002,13 +1970,6 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 		return -EBUSY;
 	}
 
-	/* Zero the gloabl flush/invalidate flags. These
-	 * will be modified as each object is bound to the
-	 * gtt
-	 */
-	dev->invalidate_domains = 0;
-	dev->flush_domains = 0;
-
 	/* Look up object handles and perform the relocations */
 	for (i = 0; i < args->buffer_count; i++) {
 		object_list[i] = drm_gem_object_lookup(dev, file_priv,
@@ -2039,10 +2000,17 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
 	i915_verify_inactive(dev, __FILE__, __LINE__);
 
+	/* Zero the global flush/invalidate flags. These
+	 * will be modified as new domains are computed
+	 * for each object
+	 */
+	dev->invalidate_domains = 0;
+	dev->flush_domains = 0;
+
 	for (i = 0; i < args->buffer_count; i++) {
 		struct drm_gem_object *obj = object_list[i];
 
-		/* Compute new gpu domains and update invalidate/flushing */
+		/* Compute new gpu domains and update invalidate/flush */
 		i915_gem_object_set_to_gpu_domain(obj,
 						  obj->pending_read_domains,
 						  obj->pending_write_domain);
@@ -2050,8 +2018,19 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
 	i915_verify_inactive(dev, __FILE__, __LINE__);
 
-	/* Flush/invalidate caches and chipset buffer */
-	flush_domains = i915_gem_dev_set_domain(dev);
+	if (dev->invalidate_domains | dev->flush_domains) {
+#if WATCH_EXEC
+		DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
+			  __func__,
+			 dev->invalidate_domains,
+			 dev->flush_domains);
+#endif
+		i915_gem_flush(dev,
+			       dev->invalidate_domains,
+			       dev->flush_domains);
+		if (dev->flush_domains)
+			(void)i915_add_request(dev, dev->flush_domains);
+	}
 
 	i915_verify_inactive(dev, __FILE__, __LINE__);
 
@@ -2071,8 +2050,6 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 			      ~0);
 #endif
 
-	(void)i915_add_request(dev, flush_domains);
-
 	/* Exec the batchbuffer */
 	ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
 	if (ret) {
-- 
cgit v1.2.1


From ac94a962b24a88ea5d00f4697550d9982f300751 Mon Sep 17 00:00:00 2001
From: Keith Packard <keithp@keithp.com>
Date: Thu, 20 Nov 2008 23:30:27 -0800
Subject: drm/i915: Retry execbuffer pinning after clearing the GTT

If we fail to pin all of the buffers in an execbuffer request, go through
and clear the GTT and try again to see if its just a matter of fragmentation

Signed-off-by: Keith Packard <keithp@keithp.com>
Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_gem.c | 57 +++++++++++++++++++++++++++++++++--------
 1 file changed, 47 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index c171a2d93bb6..174c0c3ba0b0 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1087,6 +1087,19 @@ i915_gem_evict_something(struct drm_device *dev)
 	return ret;
 }
 
+static int
+i915_gem_evict_everything(struct drm_device *dev)
+{
+	int ret;
+
+	for (;;) {
+		ret = i915_gem_evict_something(dev);
+		if (ret != 0)
+			break;
+	}
+	return ret;
+}
+
 static int
 i915_gem_object_get_page_list(struct drm_gem_object *obj)
 {
@@ -1173,7 +1186,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
 
 		ret = i915_gem_evict_something(dev);
 		if (ret != 0) {
-			DRM_ERROR("Failed to evict a buffer %d\n", ret);
+			if (ret != -ERESTARTSYS)
+				DRM_ERROR("Failed to evict a buffer %d\n", ret);
 			return ret;
 		}
 		goto search_free;
@@ -1922,6 +1936,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 	int ret, i, pinned = 0;
 	uint64_t exec_offset;
 	uint32_t seqno, flush_domains;
+	int pin_tries;
 
 #if WATCH_EXEC
 	DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
@@ -1970,7 +1985,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 		return -EBUSY;
 	}
 
-	/* Look up object handles and perform the relocations */
+	/* Look up object handles */
 	for (i = 0; i < args->buffer_count; i++) {
 		object_list[i] = drm_gem_object_lookup(dev, file_priv,
 						       exec_list[i].handle);
@@ -1980,17 +1995,39 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 			ret = -EBADF;
 			goto err;
 		}
+	}
 
-		object_list[i]->pending_read_domains = 0;
-		object_list[i]->pending_write_domain = 0;
-		ret = i915_gem_object_pin_and_relocate(object_list[i],
-						       file_priv,
-						       &exec_list[i]);
-		if (ret) {
-			DRM_ERROR("object bind and relocate failed %d\n", ret);
+	/* Pin and relocate */
+	for (pin_tries = 0; ; pin_tries++) {
+		ret = 0;
+		for (i = 0; i < args->buffer_count; i++) {
+			object_list[i]->pending_read_domains = 0;
+			object_list[i]->pending_write_domain = 0;
+			ret = i915_gem_object_pin_and_relocate(object_list[i],
+							       file_priv,
+							       &exec_list[i]);
+			if (ret)
+				break;
+			pinned = i + 1;
+		}
+		/* success */
+		if (ret == 0)
+			break;
+
+		/* error other than GTT full, or we've already tried again */
+		if (ret != -ENOMEM || pin_tries >= 1) {
+			DRM_ERROR("Failed to pin buffers %d\n", ret);
 			goto err;
 		}
-		pinned = i + 1;
+
+		/* unpin all of our buffers */
+		for (i = 0; i < pinned; i++)
+			i915_gem_object_unpin(object_list[i]);
+
+		/* evict everyone we can from the aperture */
+		ret = i915_gem_evict_everything(dev);
+		if (ret)
+			goto err;
 	}
 
 	/* Set the pending read domains for the batch buffer to COMMAND */
-- 
cgit v1.2.1


From 0235439232cb6f8a54f8976aa8330c1c98ebad0b Mon Sep 17 00:00:00 2001
From: Eric Anholt <eric@anholt.net>
Date: Wed, 26 Nov 2008 13:58:13 -0800
Subject: drm/i915: Return error in i915_gem_set_to_gtt_domain if we're not in
 the GTT.

It's only for flushing caches appropriately for GTT access, not for actually
getting it there.  Prevents potential smashing of cpu read/write domains on
unbound objects.

Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/i915/i915_gem.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 174c0c3ba0b0..3fde82be014f 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -435,6 +435,13 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 #endif
 	if (read_domains & I915_GEM_DOMAIN_GTT) {
 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
+
+		/* Silently promote "you're not bound, there was nothing to do"
+		 * to success, since the client was just asking us to
+		 * make sure everything was done.
+		 */
+		if (ret == -EINVAL)
+			ret = 0;
 	} else {
 		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 	}
@@ -1304,6 +1311,10 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
 	struct drm_i915_gem_object *obj_priv = obj->driver_private;
 	int ret;
 
+	/* Not valid to be called on unbound objects. */
+	if (obj_priv->gtt_space == NULL)
+		return -EINVAL;
+
 	i915_gem_object_flush_gpu_write_domain(obj);
 	/* Wait on any GPU rendering and flushing to occur. */
 	ret = i915_gem_object_wait_rendering(obj);
-- 
cgit v1.2.1