From 275f039db56f998aaf6e2fc74acd1f653cdcb480 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 24 Oct 2016 13:42:14 +0100 Subject: drm/i915: Move user fault tracking to a separate list We want to decouple RPM and struct_mutex, but currently RPM has to walk the list of bound objects and remove userspace mmapping before we suspend (otherwise userspace may continue to access the GTT whilst it is powered down). This currently requires the struct_mutex to walk the bound_list, but if we move that to a separate list and lock we can take the first step towards removing the struct_mutex. v2: Split runtime suspend unmapping vs regular unmapping, to make the locking (and barriers) clearer. Add the object to the userfault_list prior to inserting the first PTE, the race between add/revoke depends upon struct_mutex for regular unmappings and rpm for runtime-suspend. Signed-off-by: Chris Wilson Reviewed-by: Daniel Vetter #v1 Link: http://patchwork.freedesktop.org/patch/msgid/20161024124218.18252-1-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index dc057c770146..2d9617a3472d 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -107,7 +107,7 @@ static char get_tiling_flag(struct drm_i915_gem_object *obj) static char get_global_flag(struct drm_i915_gem_object *obj) { - return obj->fault_mappable ? 'g' : ' '; + return !list_empty(&obj->userfault_link) ? 'g' : ' '; } static char get_pin_mapped_flag(struct drm_i915_gem_object *obj) -- cgit v1.2.3 From 9c870d03674f11b49ef4f48b04d709ffa7cf1390 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 24 Oct 2016 13:42:15 +0100 Subject: drm/i915: Use RPM as the barrier for controlling user mmap access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can remove the false coupling between RPM and struct mutex by the observation that we can use the RPM wakeref as the barrier around user mmap access. That is as we tear down the user's PTE atomically from within rpm suspend and then to fault in new PTE requires the rpm wakeref, means that no user access is possible through those PTE without RPM being awake. Having made that observation, we can then remove the presumption of having to take rpm outside of struct_mutex and so allow fine grained acquisition of a wakeref around hw access rather than having to remember to acquire the wakeref early on. v2: Rejig placement of the new intel_runtime_pm_get() to be as tight as possible around the GTT pread/pwrite. Signed-off-by: Chris Wilson Cc: Imre Deak Cc: Daniel Vetter Cc: Ville Syrjälä Reviewed-by: Daniel Vetter Link: http://patchwork.freedesktop.org/patch/msgid/20161024124218.18252-2-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 56 ++++++++++++++++++++-------------- drivers/gpu/drm/i915/i915_drv.c | 19 ------------ drivers/gpu/drm/i915/i915_gem.c | 42 +++++++++++++------------ drivers/gpu/drm/i915/i915_gem_gtt.c | 17 ++++++++--- drivers/gpu/drm/i915/i915_gem_tiling.c | 4 --- 5 files changed, 69 insertions(+), 69 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 2d9617a3472d..f85e6fbf27f4 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -743,17 +743,32 @@ static int i915_interrupt_info(struct seq_file *m, void *data) I915_READ(VLV_IIR_RW)); seq_printf(m, "Display IMR:\t%08x\n", I915_READ(VLV_IMR)); - for_each_pipe(dev_priv, pipe) + for_each_pipe(dev_priv, pipe) { + enum intel_display_power_domain power_domain; + + power_domain = POWER_DOMAIN_PIPE(pipe); + if (!intel_display_power_get_if_enabled(dev_priv, + power_domain)) { + seq_printf(m, "Pipe %c power disabled\n", + pipe_name(pipe)); + continue; + } + seq_printf(m, "Pipe %c stat:\t%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); + intel_display_power_put(dev_priv, power_domain); + } + + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); seq_printf(m, "Port hotplug:\t%08x\n", I915_READ(PORT_HOTPLUG_EN)); seq_printf(m, "DPFLIPSTAT:\t%08x\n", I915_READ(VLV_DPFLIPSTAT)); seq_printf(m, "DPINVGTT:\t%08x\n", I915_READ(DPINVGTT)); + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); for (i = 0; i < 4; i++) { seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", @@ -1396,14 +1411,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) static int ironlake_drpc_info(struct seq_file *m) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; u32 rgvmodectl, rstdbyctl; u16 crstandvid; - int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); rgvmodectl = I915_READ(MEMMODECTL); @@ -1411,7 +1421,6 @@ static int ironlake_drpc_info(struct seq_file *m) crstandvid = I915_READ16(CRSTANDVID); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN)); seq_printf(m, "Boost freq: %d\n", @@ -1757,6 +1766,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) bool sr_enabled = false; intel_runtime_pm_get(dev_priv); + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); if (HAS_PCH_SPLIT(dev_priv)) sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN; @@ -1770,6 +1780,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); intel_runtime_pm_put(dev_priv); seq_printf(m, "self-refresh: %s\n", @@ -2091,12 +2102,7 @@ static const char *swizzle_string(unsigned swizzle) static int i915_swizzle_info(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; - int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); seq_printf(m, "bit6 swizzle for X-tiling = %s\n", @@ -2136,7 +2142,6 @@ static int i915_swizzle_info(struct seq_file *m, void *data) seq_puts(m, "L-shaped memory detected\n"); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); return 0; } @@ -2542,11 +2547,22 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE; else { for_each_pipe(dev_priv, pipe) { + enum transcoder cpu_transcoder = + intel_pipe_to_cpu_transcoder(dev_priv, pipe); + enum intel_display_power_domain power_domain; + + power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); + if (!intel_display_power_get_if_enabled(dev_priv, + power_domain)) + continue; + stat[pipe] = I915_READ(VLV_PSRSTAT(pipe)) & VLV_EDP_PSR_CURR_STATE_MASK; if ((stat[pipe] == VLV_EDP_PSR_ACTIVE_NORFB_UP) || (stat[pipe] == VLV_EDP_PSR_ACTIVE_SF_UPDATE)) enabled = true; + + intel_display_power_put(dev_priv, power_domain); } } @@ -3094,6 +3110,8 @@ static int i915_engine_info(struct seq_file *m, void *unused) struct intel_engine_cs *engine; enum intel_engine_id id; + intel_runtime_pm_get(dev_priv); + for_each_engine(engine, dev_priv, id) { struct intel_breadcrumbs *b = &engine->breadcrumbs; struct drm_i915_gem_request *rq; @@ -3213,6 +3231,8 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_puts(m, "\n"); } + intel_runtime_pm_put(dev_priv); + return 0; } @@ -4799,13 +4819,9 @@ i915_wedged_set(void *data, u64 val) if (i915_reset_in_progress(&dev_priv->gpu_error)) return -EAGAIN; - intel_runtime_pm_get(dev_priv); - i915_handle_error(dev_priv, val, "Manually setting wedged to %llu", val); - intel_runtime_pm_put(dev_priv); - return 0; } @@ -5040,22 +5056,16 @@ static int i915_cache_sharing_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - struct drm_device *dev = &dev_priv->drm; u32 snpcr; - int ret; if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv))) return -ENODEV; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); *val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT; diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 912d5348e3e7..885d33f341f3 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -2301,24 +2301,6 @@ static int intel_runtime_suspend(struct device *kdev) DRM_DEBUG_KMS("Suspending device\n"); - /* - * We could deadlock here in case another thread holding struct_mutex - * calls RPM suspend concurrently, since the RPM suspend will wait - * first for this RPM suspend to finish. In this case the concurrent - * RPM resume will be followed by its RPM suspend counterpart. Still - * for consistency return -EAGAIN, which will reschedule this suspend. - */ - if (!mutex_trylock(&dev->struct_mutex)) { - DRM_DEBUG_KMS("device lock contention, deffering suspend\n"); - /* - * Bump the expiration timestamp, otherwise the suspend won't - * be rescheduled. - */ - pm_runtime_mark_last_busy(kdev); - - return -EAGAIN; - } - disable_rpm_wakeref_asserts(dev_priv); /* @@ -2326,7 +2308,6 @@ static int intel_runtime_suspend(struct device *kdev) * an RPM reference. */ i915_gem_release_all_mmaps(dev_priv); - mutex_unlock(&dev->struct_mutex); intel_guc_suspend(dev); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 33ebf6d2556c..63bf51b117a9 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -826,6 +826,7 @@ i915_gem_gtt_pread(struct drm_device *dev, uint64_t offset; int ret; + intel_runtime_pm_get(to_i915(dev)); vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); if (!IS_ERR(vma)) { node.start = i915_ggtt_offset(vma); @@ -926,6 +927,7 @@ out_unpin: i915_vma_unpin(vma); } out: + intel_runtime_pm_put(to_i915(dev)); return ret; } @@ -1060,12 +1062,9 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, ret = i915_gem_shmem_pread(dev, obj, args, file); /* pread for non shmem backed objects */ - if (ret == -EFAULT || ret == -ENODEV) { - intel_runtime_pm_get(to_i915(dev)); + if (ret == -EFAULT || ret == -ENODEV) ret = i915_gem_gtt_pread(dev, obj, args->size, args->offset, args->data_ptr); - intel_runtime_pm_put(to_i915(dev)); - } i915_gem_object_put(obj); mutex_unlock(&dev->struct_mutex); @@ -1126,6 +1125,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, if (i915_gem_object_is_tiled(obj)) return -EFAULT; + intel_runtime_pm_get(i915); vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE | PIN_NONBLOCK); if (!IS_ERR(vma)) { @@ -1234,6 +1234,7 @@ out_unpin: i915_vma_unpin(vma); } out: + intel_runtime_pm_put(i915); return ret; } @@ -1466,12 +1467,12 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, * perspective, requiring manual detiling by the client. */ if (!i915_gem_object_has_struct_page(obj) || - cpu_write_needs_clflush(obj)) { - ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file); + cpu_write_needs_clflush(obj)) /* Note that the gtt paths might fail with non-page-backed user * pointers (e.g. gtt mappings when moving data between - * textures). Fallback to the shmem path in that case. */ - } + * textures). Fallback to the shmem path in that case. + */ + ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file); if (ret == -EFAULT || ret == -ENOSPC) { if (obj->phys_handle) @@ -1840,6 +1841,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) goto err_unpin; /* Mark as being mmapped into userspace for later revocation */ + assert_rpm_wakelock_held(dev_priv); spin_lock(&dev_priv->mm.userfault_lock); if (list_empty(&obj->userfault_link)) list_add(&obj->userfault_link, &dev_priv->mm.userfault_list); @@ -1925,8 +1927,13 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) /* Serialisation between user GTT access and our code depends upon * revoking the CPU's PTE whilst the mutex is held. The next user * pagefault then has to wait until we release the mutex. + * + * Note that RPM complicates somewhat by adding an additional + * requirement that operations to the GGTT be made holding the RPM + * wakeref. */ lockdep_assert_held(&i915->drm.struct_mutex); + intel_runtime_pm_get(i915); spin_lock(&i915->mm.userfault_lock); if (!list_empty(&obj->userfault_link)) { @@ -1935,7 +1942,7 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) } spin_unlock(&i915->mm.userfault_lock); if (!zap) - return; + goto out; drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->anon_inode->i_mapping); @@ -1948,6 +1955,9 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) * memory writes before touching registers / GSM. */ wmb(); + +out: + intel_runtime_pm_put(i915); } void @@ -3476,7 +3486,7 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; enum i915_cache_level level; @@ -3493,23 +3503,21 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, * cacheline, whereas normally such cachelines would get * invalidated. */ - if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) + if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) return -ENODEV; level = I915_CACHE_LLC; break; case I915_CACHING_DISPLAY: - level = HAS_WT(dev_priv) ? I915_CACHE_WT : I915_CACHE_NONE; + level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; break; default: return -EINVAL; } - intel_runtime_pm_get(dev_priv); - ret = i915_mutex_lock_interruptible(dev); if (ret) - goto rpm_put; + return ret; obj = i915_gem_object_lookup(file, args->handle); if (!obj) { @@ -3518,13 +3526,9 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, } ret = i915_gem_object_set_cache_level(obj, level); - i915_gem_object_put(obj); unlock: mutex_unlock(&dev->struct_mutex); -rpm_put: - intel_runtime_pm_put(dev_priv); - return ret; } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 062fb0ad75da..33036359c170 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -2667,6 +2667,7 @@ static int ggtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { + struct drm_i915_private *i915 = to_i915(vma->vm->dev); struct drm_i915_gem_object *obj = vma->obj; u32 pte_flags = 0; int ret; @@ -2679,8 +2680,10 @@ static int ggtt_bind_vma(struct i915_vma *vma, if (obj->gt_ro) pte_flags |= PTE_READ_ONLY; + intel_runtime_pm_get(i915); vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, cache_level, pte_flags); + intel_runtime_pm_put(i915); /* * Without aliasing PPGTT there's no difference between @@ -2696,6 +2699,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { + struct drm_i915_private *i915 = to_i915(vma->vm->dev); u32 pte_flags; int ret; @@ -2710,14 +2714,15 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, if (flags & I915_VMA_GLOBAL_BIND) { + intel_runtime_pm_get(i915); vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, cache_level, pte_flags); + intel_runtime_pm_put(i915); } if (flags & I915_VMA_LOCAL_BIND) { - struct i915_hw_ppgtt *appgtt = - to_i915(vma->vm->dev)->mm.aliasing_ppgtt; + struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; appgtt->base.insert_entries(&appgtt->base, vma->pages, vma->node.start, cache_level, pte_flags); @@ -2728,12 +2733,16 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, static void ggtt_unbind_vma(struct i915_vma *vma) { - struct i915_hw_ppgtt *appgtt = to_i915(vma->vm->dev)->mm.aliasing_ppgtt; + struct drm_i915_private *i915 = to_i915(vma->vm->dev); + struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; const u64 size = min(vma->size, vma->node.size); - if (vma->flags & I915_VMA_GLOBAL_BIND) + if (vma->flags & I915_VMA_GLOBAL_BIND) { + intel_runtime_pm_get(i915); vma->vm->clear_range(vma->vm, vma->node.start, size); + intel_runtime_pm_put(i915); + } if (vma->flags & I915_VMA_LOCAL_BIND && appgtt) appgtt->base.clear_range(&appgtt->base, diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index c21bc0068d20..71f80d2a487c 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -205,8 +205,6 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, return -EINVAL; } - intel_runtime_pm_get(dev_priv); - mutex_lock(&dev->struct_mutex); if (obj->pin_display || obj->framebuffer_references) { err = -EBUSY; @@ -302,8 +300,6 @@ err: i915_gem_object_put(obj); mutex_unlock(&dev->struct_mutex); - intel_runtime_pm_put(dev_priv); - return err; } -- cgit v1.2.3 From 0fc6a9dc94e44c8eaca364b9da03d26862d6b068 Mon Sep 17 00:00:00 2001 From: Paulo Zanoni Date: Fri, 21 Oct 2016 13:55:46 -0200 Subject: drm/i915/fbc: fix FBC_COMPRESSION_MASK on BDW+ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its size is 11:0 instead of 10:0. Found by inspecting the spec. I'm not aware of any real-world IGT failures caused by this. Signed-off-by: Paulo Zanoni Reviewed-by: Ville Syrjälä Link: http://patchwork.freedesktop.org/patch/msgid/1477065346-13736-2-git-send-email-paulo.r.zanoni@intel.com --- drivers/gpu/drm/i915/i915_debugfs.c | 10 ++++++---- drivers/gpu/drm/i915/i915_reg.h | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index f85e6fbf27f4..a5e575a7dbde 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -1683,11 +1683,13 @@ static int i915_fbc_status(struct seq_file *m, void *unused) seq_printf(m, "FBC disabled: %s\n", dev_priv->fbc.no_fbc_reason); - if (intel_fbc_is_active(dev_priv) && - INTEL_GEN(dev_priv) >= 7) + if (intel_fbc_is_active(dev_priv) && INTEL_GEN(dev_priv) >= 7) { + uint32_t mask = INTEL_GEN(dev_priv) >= 8 ? + BDW_FBC_COMPRESSION_MASK : + IVB_FBC_COMPRESSION_MASK; seq_printf(m, "Compressing: %s\n", - yesno(I915_READ(FBC_STATUS2) & - FBC_COMPRESSION_MASK)); + yesno(I915_READ(FBC_STATUS2) & mask)); + } mutex_unlock(&dev_priv->fbc.lock); intel_runtime_pm_put(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 00efaa13974d..a9be3f037f43 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -2188,8 +2188,9 @@ enum skl_disp_power_wells { #define FBC_FENCE_OFF _MMIO(0x3218) /* BSpec typo has 321Bh */ #define FBC_TAG(i) _MMIO(0x3300 + (i) * 4) -#define FBC_STATUS2 _MMIO(0x43214) -#define FBC_COMPRESSION_MASK 0x7ff +#define FBC_STATUS2 _MMIO(0x43214) +#define IVB_FBC_COMPRESSION_MASK 0x7ff +#define BDW_FBC_COMPRESSION_MASK 0xfff #define FBC_LL_SIZE (1536) -- cgit v1.2.3 From d6b40b4b179d5c3f57d1191052d4527af7321ad6 Mon Sep 17 00:00:00 2001 From: Akash Goel Date: Wed, 12 Oct 2016 21:54:29 +0530 Subject: drm/i915: New structure to contain GuC logging related fields So far there were 2 fields related to GuC logs in 'intel_guc' structure. For the support of capturing GuC logs & storing them in a local buffer, multiple new fields would have to be added. This warrants a separate structure to contain the fields related to GuC logging state. Added a new structure 'intel_guc_log' and instance of it inside 'intel_guc' structure. v2: Rebase. Signed-off-by: Akash Goel Reviewed-by: Tvrtko Ursulin Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/i915_debugfs.c | 4 ++-- drivers/gpu/drm/i915/i915_guc_submission.c | 8 ++++---- drivers/gpu/drm/i915/intel_guc.h | 8 ++++++-- drivers/gpu/drm/i915/intel_guc_loader.c | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 4751b618ae12..c5716397da20 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -2500,10 +2500,10 @@ static int i915_guc_log_dump(struct seq_file *m, void *data) struct drm_i915_gem_object *obj; int i = 0, pg; - if (!dev_priv->guc.log_vma) + if (!dev_priv->guc.log.vma) return 0; - obj = dev_priv->guc.log_vma->obj; + obj = dev_priv->guc.log.vma->obj; for (pg = 0; pg < obj->base.size / PAGE_SIZE; pg++) { u32 *log = kmap_atomic(i915_gem_object_get_page(obj, pg)); diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index a1bea7ead696..5e76b923ab4b 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -862,7 +862,7 @@ static void guc_log_create(struct intel_guc *guc) GUC_LOG_ISR_PAGES + 1 + GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT; - vma = guc->log_vma; + vma = guc->log.vma; if (!vma) { vma = guc_allocate_vma(guc, size); if (IS_ERR(vma)) { @@ -871,7 +871,7 @@ static void guc_log_create(struct intel_guc *guc) return; } - guc->log_vma = vma; + guc->log.vma = vma; } /* each allocated unit is a page */ @@ -881,7 +881,7 @@ static void guc_log_create(struct intel_guc *guc) (GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT); offset = i915_ggtt_offset(vma) >> PAGE_SHIFT; /* in pages */ - guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; + guc->log.flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; } static void guc_policies_init(struct guc_policies *policies) @@ -1065,7 +1065,7 @@ void i915_guc_submission_fini(struct drm_i915_private *dev_priv) struct intel_guc *guc = &dev_priv->guc; i915_vma_unpin_and_release(&guc->ads_vma); - i915_vma_unpin_and_release(&guc->log_vma); + i915_vma_unpin_and_release(&guc->log.vma); if (guc->ctx_pool_vma) ida_destroy(&guc->ctx_ids); diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h index 5cdf7aa75be5..c73294164fed 100644 --- a/drivers/gpu/drm/i915/intel_guc.h +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -123,10 +123,14 @@ struct intel_guc_fw { uint32_t ucode_offset; }; +struct intel_guc_log { + uint32_t flags; + struct i915_vma *vma; +}; + struct intel_guc { struct intel_guc_fw guc_fw; - uint32_t log_flags; - struct i915_vma *log_vma; + struct intel_guc_log log; struct i915_vma *ads_vma; struct i915_vma *ctx_pool_vma; diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c index 740be0b2a360..a75abb8cc0cc 100644 --- a/drivers/gpu/drm/i915/intel_guc_loader.c +++ b/drivers/gpu/drm/i915/intel_guc_loader.c @@ -211,7 +211,7 @@ static void guc_params_init(struct drm_i915_private *dev_priv) params[GUC_CTL_FEATURE] |= GUC_CTL_DISABLE_SCHEDULER | GUC_CTL_VCS2_ENABLED; - params[GUC_CTL_LOG_PARAMS] = guc->log_flags; + params[GUC_CTL_LOG_PARAMS] = guc->log.flags; if (i915.guc_log_level >= 0) { params[GUC_CTL_DEBUG] = -- cgit v1.2.3 From 5aa1ee4b12bee127576b3ea41eeafda2c45bc118 Mon Sep 17 00:00:00 2001 From: Akash Goel Date: Wed, 12 Oct 2016 21:54:36 +0530 Subject: drm/i915: Add stats for GuC log buffer flush interrupts GuC firmware sends an interrupt to flush the log buffer when it becomes half full. GuC firmware also tracks how many times the buffer overflowed. It would be useful to maintain a statistics of how many flush interrupts were received and for which type of log buffer, along with the overflow count of each buffer type. Augmented i915_log_info debugfs to report back these statistics. v2: - Update the logic to detect multiple overflows between the 2 flush interrupts and also log a message for overflow (Tvrtko) - Track the number of times there was no free sub buffer to capture the GuC log buffer. (Tvrtko) v3: - Fix the printf field width for overflow counter, set it to 10 as per the max value of u32, which takes 10 digits in decimal form. (Tvrtko) v4: - Move the log buffer overflow handling to a new function for better readability. (Tvrtko) Signed-off-by: Akash Goel Reviewed-by: Tvrtko Ursulin Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/i915_debugfs.c | 28 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_guc_submission.c | 31 +++++++++++++++++++++++++++++- drivers/gpu/drm/i915/i915_irq.c | 2 ++ drivers/gpu/drm/i915/intel_guc.h | 7 +++++++ 4 files changed, 67 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index c5716397da20..e24f69cdc69b 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -2416,6 +2416,32 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data) return 0; } +static void i915_guc_log_info(struct seq_file *m, + struct drm_i915_private *dev_priv) +{ + struct intel_guc *guc = &dev_priv->guc; + + seq_puts(m, "\nGuC logging stats:\n"); + + seq_printf(m, "\tISR: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_ISR_LOG_BUFFER], + guc->log.total_overflow_count[GUC_ISR_LOG_BUFFER]); + + seq_printf(m, "\tDPC: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_DPC_LOG_BUFFER], + guc->log.total_overflow_count[GUC_DPC_LOG_BUFFER]); + + seq_printf(m, "\tCRASH: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_CRASH_DUMP_LOG_BUFFER], + guc->log.total_overflow_count[GUC_CRASH_DUMP_LOG_BUFFER]); + + seq_printf(m, "\tTotal flush interrupt count: %u\n", + guc->log.flush_interrupt_count); + + seq_printf(m, "\tCapture miss count: %u\n", + guc->log.capture_miss_count); +} + static void i915_guc_client_info(struct seq_file *m, struct drm_i915_private *dev_priv, struct i915_guc_client *client) @@ -2489,6 +2515,8 @@ static int i915_guc_info(struct seq_file *m, void *data) seq_printf(m, "\nGuC execbuf client @ %p:\n", guc.execbuf_client); i915_guc_client_info(m, dev_priv, &client); + i915_guc_log_info(m, dev_priv); + /* Add more as required ... */ return 0; diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index cafff7ca71c7..f2d71cadae99 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -1016,6 +1016,29 @@ static void *guc_get_write_buffer(struct intel_guc *guc) return relay_reserve(guc->log.relay_chan, 0); } +static bool +guc_check_log_buf_overflow(struct intel_guc *guc, + enum guc_log_buffer_type type, unsigned int full_cnt) +{ + unsigned int prev_full_cnt = guc->log.prev_overflow_count[type]; + bool overflow = false; + + if (full_cnt != prev_full_cnt) { + overflow = true; + + guc->log.prev_overflow_count[type] = full_cnt; + guc->log.total_overflow_count[type] += full_cnt - prev_full_cnt; + + if (full_cnt < prev_full_cnt) { + /* buffer_full_cnt is a 4 bit counter */ + guc->log.total_overflow_count[type] += 16; + } + DRM_ERROR_RATELIMITED("GuC log buffer overflow\n"); + } + + return overflow; +} + static unsigned int guc_get_log_buffer_size(enum guc_log_buffer_type type) { switch (type) { @@ -1036,7 +1059,7 @@ static void guc_read_update_log_buffer(struct intel_guc *guc) { struct guc_log_buffer_state *log_buf_state, *log_buf_snapshot_state; struct guc_log_buffer_state log_buf_state_local; - unsigned int buffer_size, write_offset; + unsigned int buffer_size, write_offset, full_cnt; enum guc_log_buffer_type type; void *src_data, *dst_data; @@ -1062,6 +1085,11 @@ static void guc_read_update_log_buffer(struct intel_guc *guc) sizeof(struct guc_log_buffer_state)); buffer_size = guc_get_log_buffer_size(type); write_offset = log_buf_state_local.sampled_write_ptr; + full_cnt = log_buf_state_local.buffer_full_cnt; + + /* Bookkeeping stuff */ + guc->log.flush_count[type] += log_buf_state_local.flush_to_file; + guc_check_log_buf_overflow(guc, type, full_cnt); /* Update the state of shared log buffer */ log_buf_state->read_ptr = write_offset; @@ -1099,6 +1127,7 @@ static void guc_read_update_log_buffer(struct intel_guc *guc) * getting consumed by User at a slow rate. */ DRM_ERROR_RATELIMITED("no sub-buffer to capture logs\n"); + guc->log.capture_miss_count++; } } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 253c22109c3f..9111cfdf217a 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -1691,6 +1691,8 @@ static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) /* Handle flush interrupt in bottom half */ queue_work(dev_priv->guc.log.flush_wq, &dev_priv->guc.log.flush_work); + + dev_priv->guc.log.flush_interrupt_count++; } else { /* Not clearing of unhandled event bits won't result in * re-triggering of the interrupt. diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h index 907d13a71c23..d034c9845a22 100644 --- a/drivers/gpu/drm/i915/intel_guc.h +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -130,6 +130,13 @@ struct intel_guc_log { struct workqueue_struct *flush_wq; struct work_struct flush_work; struct rchan *relay_chan; + + /* logging related stats */ + u32 capture_miss_count; + u32 flush_interrupt_count; + u32 prev_overflow_count[GUC_MAX_LOG_BUFFER]; + u32 total_overflow_count[GUC_MAX_LOG_BUFFER]; + u32 flush_count[GUC_MAX_LOG_BUFFER]; }; struct intel_guc { -- cgit v1.2.3 From 685534ef4c8faff1f69c9962ad3013e17354ff8b Mon Sep 17 00:00:00 2001 From: Sagar Arun Kamble Date: Wed, 12 Oct 2016 21:54:41 +0530 Subject: drm/i915: Debugfs support for GuC logging control This patch provides debugfs interface i915_guc_output_control for on the fly enabling/disabling of logging in GuC firmware and controlling the verbosity level of logs. The value written to the file, should have bit 0 set to enable logging and bits 4-7 should contain the verbosity info. v2: Add a forceful flush, to collect left over logs, on disabling logging. Useful for Validation. v3: Besides minor cleanup, implement read method for the debugfs file and set the guc_log_level to -1 when logging is disabled. (Tvrtko) v4: Minor cleanup & rebase. (Tvrtko) v5: - Lock struct_mutex after the NULL check for guc log buffer vma. (Chris) - Rebase. Signed-off-by: Sagar Arun Kamble Signed-off-by: Akash Goel Reviewed-by: Tvrtko Ursulin Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/i915_debugfs.c | 41 ++++++++++++++++++++- drivers/gpu/drm/i915/i915_guc_submission.c | 59 ++++++++++++++++++++++++++++++ drivers/gpu/drm/i915/intel_guc.h | 1 + 3 files changed, 100 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index e24f69cdc69b..be92efe3453f 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -2548,6 +2548,44 @@ static int i915_guc_log_dump(struct seq_file *m, void *data) return 0; } +static int i915_guc_log_control_get(void *data, u64 *val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = to_i915(dev); + + if (!dev_priv->guc.log.vma) + return -EINVAL; + + *val = i915.guc_log_level; + + return 0; +} + +static int i915_guc_log_control_set(void *data, u64 val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = to_i915(dev); + int ret; + + if (!dev_priv->guc.log.vma) + return -EINVAL; + + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; + + intel_runtime_pm_get(dev_priv); + ret = i915_guc_log_control(dev_priv, val); + intel_runtime_pm_put(dev_priv); + + mutex_unlock(&dev->struct_mutex); + return ret; +} + +DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_control_fops, + i915_guc_log_control_get, i915_guc_log_control_set, + "%lld\n"); + static int i915_edp_psr_status(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); @@ -5454,7 +5492,8 @@ static const struct i915_debugfs_files { {"i915_fbc_false_color", &i915_fbc_fc_fops}, {"i915_dp_test_data", &i915_displayport_test_data_fops}, {"i915_dp_test_type", &i915_displayport_test_type_fops}, - {"i915_dp_test_active", &i915_displayport_test_active_fops} + {"i915_dp_test_active", &i915_displayport_test_active_fops}, + {"i915_guc_log_control", &i915_guc_log_control_fops} }; void intel_display_crc_init(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index 9fd9a3ae1229..6fac8e9cbd38 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -193,6 +193,16 @@ static int host2guc_force_logbuffer_flush(struct intel_guc *guc) return host2guc_action(guc, data, 2); } +static int host2guc_logging_control(struct intel_guc *guc, u32 control_val) +{ + u32 data[2]; + + data[0] = HOST2GUC_ACTION_UK_LOG_ENABLE_LOGGING; + data[1] = control_val; + + return host2guc_action(guc, data, 2); +} + /* * Initialise, update, or clear doorbell data shared with the GuC * @@ -1605,3 +1615,52 @@ void i915_guc_register(struct drm_i915_private *dev_priv) guc_log_late_setup(&dev_priv->guc); mutex_unlock(&dev_priv->drm.struct_mutex); } + +int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val) +{ + union guc_log_control log_param; + int ret; + + log_param.value = control_val; + + if (log_param.verbosity < GUC_LOG_VERBOSITY_MIN || + log_param.verbosity > GUC_LOG_VERBOSITY_MAX) + return -EINVAL; + + /* This combination doesn't make sense & won't have any effect */ + if (!log_param.logging_enabled && (i915.guc_log_level < 0)) + return 0; + + ret = host2guc_logging_control(&dev_priv->guc, log_param.value); + if (ret < 0) { + DRM_DEBUG_DRIVER("host2guc action failed %d\n", ret); + return ret; + } + + i915.guc_log_level = log_param.verbosity; + + /* If log_level was set as -1 at boot time, then the relay channel file + * wouldn't have been created by now and interrupts also would not have + * been enabled. + */ + if (!dev_priv->guc.log.relay_chan) { + ret = guc_log_late_setup(&dev_priv->guc); + if (!ret) + gen9_enable_guc_interrupts(dev_priv); + } else if (!log_param.logging_enabled) { + /* Once logging is disabled, GuC won't generate logs & send an + * interrupt. But there could be some data in the log buffer + * which is yet to be captured. So request GuC to update the log + * buffer state and then collect the left over logs. + */ + i915_guc_flush_logs(dev_priv); + + /* As logging is disabled, update log level to reflect that */ + i915.guc_log_level = -1; + } else { + /* In case interrupts were disabled, enable them now */ + gen9_enable_guc_interrupts(dev_priv); + } + + return ret; +} diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h index 71da75e7f6b4..49ced0bad0f5 100644 --- a/drivers/gpu/drm/i915/intel_guc.h +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -188,5 +188,6 @@ void i915_guc_capture_logs(struct drm_i915_private *dev_priv); void i915_guc_flush_logs(struct drm_i915_private *dev_priv); void i915_guc_register(struct drm_i915_private *dev_priv); void i915_guc_unregister(struct drm_i915_private *dev_priv); +int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val); #endif -- cgit v1.2.3 From 8b364b41ced2ac8eea9958ea084f10289a85419b Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Wed, 26 Oct 2016 15:51:28 -0700 Subject: drm/i915: Rename for_each_plane -> for_each_universal_plane This macro's name is a bit misleading; it doesn't actually iterate over all planes since it omits the cursor plane. Its only uses are in gen9 code which is using it to iterate over the universal planes (which we treat as primary+sprites); in these cases the legacy cursor registers are programmed independently if necessary. The macro's iterator value (0 for primary plane, spritenum+1 for each secondary plane) also isn't meaningful outside the gen9 context where the hardware considers them to all be "universal" planes that follow this numbering. This is just a renaming/clarification patch with no functional change. However it will make the subsequent patches more clear. Signed-off-by: Matt Roper Link: http://patchwork.freedesktop.org/patch/msgid/1477522291-10874-2-git-send-email-matthew.d.roper@intel.com Reviewed-by: Paulo Zanoni --- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_drv.h | 2 +- drivers/gpu/drm/i915/intel_display.c | 2 +- drivers/gpu/drm/i915/intel_pm.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index be92efe3453f..9f5a39209f52 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -3463,7 +3463,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused) for_each_pipe(dev_priv, pipe) { seq_printf(m, "Pipe %c\n", pipe_name(pipe)); - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { entry = &ddb->plane[pipe][plane]; seq_printf(m, " Plane%-8d%8u%8u%8u\n", plane + 1, entry->start, entry->end, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index a8f006e3b45d..930503aac46d 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -312,7 +312,7 @@ struct i915_hotplug { #define for_each_pipe_masked(__dev_priv, __p, __mask) \ for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \ for_each_if ((__mask) & (1 << (__p))) -#define for_each_plane(__dev_priv, __pipe, __p) \ +#define for_each_universal_plane(__dev_priv, __pipe, __p) \ for ((__p) = 0; \ (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ (__p)++) diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 895b3dc50e3f..cb7dd11277fd 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -13485,7 +13485,7 @@ static void verify_wm_state(struct drm_crtc *crtc, sw_ddb = &dev_priv->wm.skl_hw.ddb; /* planes */ - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { hw_plane_wm = &hw_wm.planes[plane]; sw_plane_wm = &sw_wm->planes[plane]; diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 9e0e8741253e..58d3ba05b219 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -3160,7 +3160,7 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) continue; - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { val = I915_READ(PLANE_BUF_CFG(pipe, plane)); skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane], val); -- cgit v1.2.3 From a4f5ea64f0a818586b9de71803824b43dd01e517 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:35 +0100 Subject: drm/i915: Refactor object page API The plan is to make obtaining the backing storage for the object avoid struct_mutex (i.e. use its own locking). The first step is to update the API so that normal users only call pin/unpin whilst working on the backing storage. Signed-off-by: Chris Wilson Reviewed-by: Tvrtko Ursulin Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-12-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_cmd_parser.c | 2 +- drivers/gpu/drm/i915/i915_debugfs.c | 17 +-- drivers/gpu/drm/i915/i915_drv.h | 94 ++++++++---- drivers/gpu/drm/i915/i915_gem.c | 205 +++++++++++++-------------- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 3 +- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 14 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 +- drivers/gpu/drm/i915/i915_gem_fence.c | 4 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 10 +- drivers/gpu/drm/i915/i915_gem_internal.c | 12 +- drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/i915_gem_shrinker.c | 10 +- drivers/gpu/drm/i915/i915_gem_stolen.c | 24 ++-- drivers/gpu/drm/i915/i915_gem_tiling.c | 8 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 30 ++-- drivers/gpu/drm/i915/i915_gpu_error.c | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 6 +- 17 files changed, 233 insertions(+), 214 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index f191d7b66b1d..f5039f4f988f 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, } if (ret == 0 && needs_clflush_after) - drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len); + drm_clflush_virt_range(shadow_batch_obj->mm.mapping, batch_len); i915_gem_object_unpin_map(shadow_batch_obj); return ret; diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 9f5a39209f52..e97a16cb6b67 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -112,7 +112,7 @@ static char get_global_flag(struct drm_i915_gem_object *obj) static char get_pin_mapped_flag(struct drm_i915_gem_object *obj) { - return obj->mapping ? 'M' : ' '; + return obj->mm.mapping ? 'M' : ' '; } static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj) @@ -158,8 +158,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) i915_gem_active_get_seqno(&obj->last_write, &obj->base.dev->struct_mutex), i915_cache_level_str(dev_priv, obj->cache_level), - obj->dirty ? " dirty" : "", - obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); + obj->mm.dirty ? " dirty" : "", + obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : ""); if (obj->base.name) seq_printf(m, " (name: %d)", obj->base.name); list_for_each_entry(vma, &obj->vma_list, obj_link) { @@ -403,12 +403,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) size += obj->base.size; ++count; - if (obj->madv == I915_MADV_DONTNEED) { + if (obj->mm.madv == I915_MADV_DONTNEED) { purgeable_size += obj->base.size; ++purgeable_count; } - if (obj->mapping) { + if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; } @@ -425,12 +425,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) ++dpy_count; } - if (obj->madv == I915_MADV_DONTNEED) { + if (obj->mm.madv == I915_MADV_DONTNEED) { purgeable_size += obj->base.size; ++purgeable_count; } - if (obj->mapping) { + if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; } @@ -2028,7 +2028,7 @@ static void i915_dump_lrc_obj(struct seq_file *m, seq_printf(m, "\tBound in GGTT at 0x%08x\n", i915_ggtt_offset(vma)); - if (i915_gem_object_get_pages(vma->obj)) { + if (i915_gem_object_pin_pages(vma->obj)) { seq_puts(m, "\tFailed to get pages for context object\n\n"); return; } @@ -2047,6 +2047,7 @@ static void i915_dump_lrc_obj(struct seq_file *m, kunmap_atomic(reg_state); } + i915_gem_object_unpin_pages(vma->obj); seq_putc(m, '\n'); } diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 85ae83a0c937..50781cbe74d0 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -2252,17 +2252,6 @@ struct drm_i915_gem_object { */ #define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES) - /** - * This is set if the object has been written to since last bound - * to the GTT - */ - unsigned int dirty:1; - - /** - * Advice: are the backing pages purgeable? - */ - unsigned int madv:2; - /* * Is the object to be mapped as read-only to the GPU * Only honoured if hardware has relevant pte bit @@ -2284,16 +2273,31 @@ struct drm_i915_gem_object { unsigned int bind_count; unsigned int pin_display; - struct sg_table *pages; - int pages_pin_count; - struct i915_gem_object_page_iter { - struct scatterlist *sg_pos; - unsigned int sg_idx; /* in pages, but 32bit eek! */ + struct { + unsigned int pages_pin_count; + + struct sg_table *pages; + void *mapping; - struct radix_tree_root radix; - struct mutex lock; /* protects this cache */ - } get_page; - void *mapping; + struct i915_gem_object_page_iter { + struct scatterlist *sg_pos; + unsigned int sg_idx; /* in pages, but 32bit eek! */ + + struct radix_tree_root radix; + struct mutex lock; /* protects this cache */ + } get_page; + + /** + * Advice: are the backing pages purgeable? + */ + unsigned int madv:2; + + /** + * This is set if the object has been written to since the + * pages were last acquired. + */ + bool dirty:1; + } mm; /** Breadcrumb of last rendering to the buffer. * There can only be one writer, but we allow for multiple readers. @@ -3182,14 +3186,11 @@ void i915_vma_close(struct i915_vma *vma); void i915_vma_destroy(struct i915_vma *vma); int i915_gem_object_unbind(struct drm_i915_gem_object *obj); -int i915_gem_object_put_pages(struct drm_i915_gem_object *obj); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); -int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj); - -static inline int __sg_page_count(struct scatterlist *sg) +static inline int __sg_page_count(const struct scatterlist *sg) { return sg->length >> PAGE_SHIFT; } @@ -3210,19 +3211,52 @@ dma_addr_t i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, unsigned long n); -static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) +int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); + +static inline int __must_check +i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) +{ + lockdep_assert_held(&obj->base.dev->struct_mutex); + + if (obj->mm.pages_pin_count++) + return 0; + + return __i915_gem_object_get_pages(obj); +} + +static inline void +__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { - GEM_BUG_ON(obj->pages == NULL); - obj->pages_pin_count++; + lockdep_assert_held(&obj->base.dev->struct_mutex); + GEM_BUG_ON(!obj->mm.pages); + + obj->mm.pages_pin_count++; +} + +static inline bool +i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) +{ + return obj->mm.pages_pin_count; +} + +static inline void +__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) +{ + lockdep_assert_held(&obj->base.dev->struct_mutex); + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + GEM_BUG_ON(!obj->mm.pages); + + obj->mm.pages_pin_count--; + GEM_BUG_ON(obj->mm.pages_pin_count < obj->bind_count); } static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { - GEM_BUG_ON(obj->pages_pin_count == 0); - obj->pages_pin_count--; - GEM_BUG_ON(obj->pages_pin_count < obj->bind_count); + __i915_gem_object_unpin_pages(obj); } +int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj); + enum i915_map_type { I915_MAP_WB = 0, I915_MAP_WC, diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index aa0de3aa2565..0d702c8f7b7b 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -216,7 +216,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) sg_dma_address(sg) = obj->phys_handle->busaddr; sg_dma_len(sg) = obj->base.size; - obj->pages = st; + obj->mm.pages = st; return 0; } @@ -225,7 +225,7 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) { int ret; - BUG_ON(obj->madv == __I915_MADV_PURGED); + GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); ret = i915_gem_object_set_to_cpu_domain(obj, true); if (WARN_ON(ret)) { @@ -235,10 +235,10 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; } - if (obj->madv == I915_MADV_DONTNEED) - obj->dirty = 0; + if (obj->mm.madv == I915_MADV_DONTNEED) + obj->mm.dirty = false; - if (obj->dirty) { + if (obj->mm.dirty) { struct address_space *mapping = obj->base.filp->f_mapping; char *vaddr = obj->phys_handle->vaddr; int i; @@ -257,22 +257,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) kunmap_atomic(dst); set_page_dirty(page); - if (obj->madv == I915_MADV_WILLNEED) + if (obj->mm.madv == I915_MADV_WILLNEED) mark_page_accessed(page); put_page(page); vaddr += PAGE_SIZE; } - obj->dirty = 0; + obj->mm.dirty = false; } - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(obj->mm.pages); + kfree(obj->mm.pages); } static void i915_gem_object_release_phys(struct drm_i915_gem_object *obj) { drm_pci_free(obj->base.dev, obj->phys_handle); + i915_gem_object_unpin_pages(obj); } static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { @@ -507,7 +508,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, return 0; } - if (obj->madv != I915_MADV_WILLNEED) + if (obj->mm.madv != I915_MADV_WILLNEED) return -EFAULT; if (obj->base.filp == NULL) @@ -517,7 +518,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, if (ret) return ret; - ret = i915_gem_object_put_pages(obj); + ret = __i915_gem_object_put_pages(obj); if (ret) return ret; @@ -529,7 +530,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, obj->phys_handle = phys; obj->ops = &i915_gem_phys_ops; - return i915_gem_object_get_pages(obj); + return i915_gem_object_pin_pages(obj); } static int @@ -725,12 +726,10 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, if (ret) return ret; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - i915_gem_object_flush_gtt_write_domain(obj); /* If we're not in the cpu read domain, set ourself into the gtt @@ -778,12 +777,10 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, if (ret) return ret; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - i915_gem_object_flush_gtt_write_domain(obj); /* If we're not in the cpu write domain, set ourself into the @@ -813,7 +810,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, obj->cache_dirty = true; intel_fb_obj_invalidate(obj, ORIGIN_CPU); - obj->dirty = 1; + obj->mm.dirty = true; /* return with the pages pinned */ return 0; @@ -951,13 +948,11 @@ i915_gem_gtt_pread(struct drm_device *dev, if (ret) goto out; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) { remove_mappable_node(&node); goto out; } - - i915_gem_object_pin_pages(obj); } ret = i915_gem_object_set_to_gtt_domain(obj, false); @@ -1064,7 +1059,7 @@ i915_gem_shmem_pread(struct drm_device *dev, offset = args->offset; remain = args->size; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, + for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, offset >> PAGE_SHIFT) { struct page *page = sg_page_iter_page(&sg_iter); @@ -1254,13 +1249,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, if (ret) goto out; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) { remove_mappable_node(&node); goto out; } - - i915_gem_object_pin_pages(obj); } ret = i915_gem_object_set_to_gtt_domain(obj, true); @@ -1268,7 +1261,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, goto out_unpin; intel_fb_obj_invalidate(obj, ORIGIN_CPU); - obj->dirty = true; + obj->mm.dirty = true; user_data = u64_to_user_ptr(args->data_ptr); offset = args->offset; @@ -1439,7 +1432,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev, offset = args->offset; remain = args->size; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, + for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, offset >> PAGE_SHIFT) { struct page *page = sg_page_iter_page(&sg_iter); int partial_cacheline_write; @@ -2266,7 +2259,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj) * backing pages, *now*. */ shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); - obj->madv = __I915_MADV_PURGED; + obj->mm.madv = __I915_MADV_PURGED; } /* Try to discard unwanted pages */ @@ -2275,7 +2268,7 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj) { struct address_space *mapping; - switch (obj->madv) { + switch (obj->mm.madv) { case I915_MADV_DONTNEED: i915_gem_object_truncate(obj); case __I915_MADV_PURGED: @@ -2296,7 +2289,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) struct page *page; int ret; - BUG_ON(obj->madv == __I915_MADV_PURGED); + GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); ret = i915_gem_object_set_to_cpu_domain(obj, true); if (WARN_ON(ret)) { @@ -2312,22 +2305,22 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) if (i915_gem_object_needs_bit17_swizzle(obj)) i915_gem_object_save_bit_17_swizzle(obj); - if (obj->madv == I915_MADV_DONTNEED) - obj->dirty = 0; + if (obj->mm.madv == I915_MADV_DONTNEED) + obj->mm.dirty = false; - for_each_sgt_page(page, sgt_iter, obj->pages) { - if (obj->dirty) + for_each_sgt_page(page, sgt_iter, obj->mm.pages) { + if (obj->mm.dirty) set_page_dirty(page); - if (obj->madv == I915_MADV_WILLNEED) + if (obj->mm.madv == I915_MADV_WILLNEED) mark_page_accessed(page); put_page(page); } - obj->dirty = 0; + obj->mm.dirty = false; - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(obj->mm.pages); + kfree(obj->mm.pages); } static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) @@ -2335,21 +2328,20 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) struct radix_tree_iter iter; void **slot; - radix_tree_for_each_slot(slot, &obj->get_page.radix, &iter, 0) - radix_tree_delete(&obj->get_page.radix, iter.index); + radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) + radix_tree_delete(&obj->mm.get_page.radix, iter.index); } -int -i915_gem_object_put_pages(struct drm_i915_gem_object *obj) +int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) { const struct drm_i915_gem_object_ops *ops = obj->ops; lockdep_assert_held(&obj->base.dev->struct_mutex); - if (obj->pages == NULL) + if (!obj->mm.pages) return 0; - if (obj->pages_pin_count) + if (i915_gem_object_has_pinned_pages(obj)) return -EBUSY; GEM_BUG_ON(obj->bind_count); @@ -2359,22 +2351,22 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj) * lists early. */ list_del(&obj->global_list); - if (obj->mapping) { + if (obj->mm.mapping) { void *ptr; - ptr = ptr_mask_bits(obj->mapping); + ptr = ptr_mask_bits(obj->mm.mapping); if (is_vmalloc_addr(ptr)) vunmap(ptr); else kunmap(kmap_to_page(ptr)); - obj->mapping = NULL; + obj->mm.mapping = NULL; } __i915_gem_object_reset_page_iter(obj); ops->put_pages(obj); - obj->pages = NULL; + obj->mm.pages = NULL; i915_gem_object_invalidate(obj); @@ -2474,7 +2466,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) } if (sg) /* loop terminated early; short sg table */ sg_mark_end(sg); - obj->pages = st; + obj->mm.pages = st; ret = i915_gem_gtt_prepare_object(obj); if (ret) @@ -2485,7 +2477,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) if (i915_gem_object_is_tiled(obj) && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) - i915_gem_object_pin_pages(obj); + __i915_gem_object_pin_pages(obj); return 0; @@ -2517,8 +2509,7 @@ err_pages: * either as a result of memory pressure (reaping pages under the shrinker) * or as the object is itself released. */ -int -i915_gem_object_get_pages(struct drm_i915_gem_object *obj) +int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); const struct drm_i915_gem_object_ops *ops = obj->ops; @@ -2526,24 +2517,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj) lockdep_assert_held(&obj->base.dev->struct_mutex); - if (obj->pages) + if (obj->mm.pages) return 0; - if (obj->madv != I915_MADV_WILLNEED) { + if (obj->mm.madv != I915_MADV_WILLNEED) { DRM_DEBUG("Attempting to obtain a purgeable object\n"); + __i915_gem_object_unpin_pages(obj); return -EFAULT; } - BUG_ON(obj->pages_pin_count); - ret = ops->get_pages(obj); - if (ret) + if (ret) { + __i915_gem_object_unpin_pages(obj); return ret; + } list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx = 0; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx = 0; return 0; } @@ -2553,7 +2545,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, enum i915_map_type type) { unsigned long n_pages = obj->base.size >> PAGE_SHIFT; - struct sg_table *sgt = obj->pages; + struct sg_table *sgt = obj->mm.pages; struct sgt_iter sgt_iter; struct page *page; struct page *stack_pages[32]; @@ -2607,14 +2599,13 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, lockdep_assert_held(&obj->base.dev->struct_mutex); GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ERR_PTR(ret); - i915_gem_object_pin_pages(obj); - pinned = obj->pages_pin_count > 1; + pinned = obj->mm.pages_pin_count > 1; - ptr = ptr_unpack_bits(obj->mapping, has_type); + ptr = ptr_unpack_bits(obj->mm.mapping, has_type); if (ptr && has_type != type) { if (pinned) { ret = -EBUSY; @@ -2626,7 +2617,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, else kunmap(kmap_to_page(ptr)); - ptr = obj->mapping = NULL; + ptr = obj->mm.mapping = NULL; } if (!ptr) { @@ -2636,7 +2627,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, goto err; } - obj->mapping = ptr_pack_bits(ptr, type); + obj->mm.mapping = ptr_pack_bits(ptr, type); } return ptr; @@ -3087,7 +3078,7 @@ int i915_vma_unbind(struct i915_vma *vma) goto destroy; GEM_BUG_ON(obj->bind_count == 0); - GEM_BUG_ON(!obj->pages); + GEM_BUG_ON(!obj->mm.pages); if (i915_vma_is_map_and_fenceable(vma)) { /* release the fence reg _after_ flushing */ @@ -3111,7 +3102,7 @@ int i915_vma_unbind(struct i915_vma *vma) drm_mm_remove_node(&vma->node); list_move_tail(&vma->vm_link, &vma->vm->unbound_list); - if (vma->pages != obj->pages) { + if (vma->pages != obj->mm.pages) { GEM_BUG_ON(!vma->pages); sg_free_table(vma->pages); kfree(vma->pages); @@ -3244,12 +3235,10 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) return -E2BIG; } - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - if (flags & PIN_OFFSET_FIXED) { u64 offset = flags & PIN_OFFSET_MASK; if (offset & (alignment - 1) || offset > end - size) { @@ -3331,7 +3320,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj, * to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ - if (obj->pages == NULL) + if (!obj->mm.pages) return false; /* @@ -3355,7 +3344,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj, } trace_i915_gem_object_clflush(obj); - drm_clflush_sg(obj->pages); + drm_clflush_sg(obj->mm.pages); obj->cache_dirty = false; return true; @@ -3469,7 +3458,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) * continue to assume that the obj remained out of the CPU cached * domain. */ - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; @@ -3493,7 +3482,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) if (write) { obj->base.read_domains = I915_GEM_DOMAIN_GTT; obj->base.write_domain = I915_GEM_DOMAIN_GTT; - obj->dirty = 1; + obj->mm.dirty = true; } trace_i915_gem_object_change_domain(obj, @@ -3502,6 +3491,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) /* And bump the LRU for this access */ i915_gem_object_bump_inactive_ggtt(obj); + i915_gem_object_unpin_pages(obj); return 0; } @@ -4304,23 +4294,23 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, goto unlock; } - if (obj->pages && + if (obj->mm.pages && i915_gem_object_is_tiled(obj) && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { - if (obj->madv == I915_MADV_WILLNEED) - i915_gem_object_unpin_pages(obj); + if (obj->mm.madv == I915_MADV_WILLNEED) + __i915_gem_object_unpin_pages(obj); if (args->madv == I915_MADV_WILLNEED) - i915_gem_object_pin_pages(obj); + __i915_gem_object_pin_pages(obj); } - if (obj->madv != __I915_MADV_PURGED) - obj->madv = args->madv; + if (obj->mm.madv != __I915_MADV_PURGED) + obj->mm.madv = args->madv; /* if the object is no longer attached, discard its backing storage */ - if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) + if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages) i915_gem_object_truncate(obj); - args->retained = obj->madv != __I915_MADV_PURGED; + args->retained = obj->mm.madv != __I915_MADV_PURGED; i915_gem_object_put(obj); unlock: @@ -4347,9 +4337,10 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->ops = ops; obj->frontbuffer_ggtt_origin = ORIGIN_GTT; - obj->madv = I915_MADV_WILLNEED; - INIT_RADIX_TREE(&obj->get_page.radix, GFP_KERNEL | __GFP_NOWARN); - mutex_init(&obj->get_page.lock); + + obj->mm.madv = I915_MADV_WILLNEED; + INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); + mutex_init(&obj->mm.get_page.lock); i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size); } @@ -4441,7 +4432,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj) * back the contents from the GPU. */ - if (obj->madv != I915_MADV_WILLNEED) + if (obj->mm.madv != I915_MADV_WILLNEED) return false; if (obj->base.filp == NULL) @@ -4483,32 +4474,27 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) } GEM_BUG_ON(obj->bind_count); - /* Stolen objects don't hold a ref, but do hold pin count. Fix that up - * before progressing. */ - if (obj->stolen) - i915_gem_object_unpin_pages(obj); - WARN_ON(atomic_read(&obj->frontbuffer_bits)); - if (obj->pages && obj->madv == I915_MADV_WILLNEED && + if (obj->mm.pages && obj->mm.madv == I915_MADV_WILLNEED && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES && i915_gem_object_is_tiled(obj)) - i915_gem_object_unpin_pages(obj); + __i915_gem_object_unpin_pages(obj); - if (WARN_ON(obj->pages_pin_count)) - obj->pages_pin_count = 0; + if (obj->ops->release) + obj->ops->release(obj); + + if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) + obj->mm.pages_pin_count = 0; if (discard_backing_storage(obj)) - obj->madv = I915_MADV_DONTNEED; - i915_gem_object_put_pages(obj); + obj->mm.madv = I915_MADV_DONTNEED; + __i915_gem_object_put_pages(obj); - BUG_ON(obj->pages); + GEM_BUG_ON(obj->mm.pages); if (obj->base.import_attach) drm_prime_gem_destroy(&obj->base, NULL); - if (obj->ops->release) - obj->ops->release(obj); - drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(dev_priv, obj->base.size); @@ -5063,14 +5049,13 @@ i915_gem_object_create_from_data(struct drm_device *dev, if (ret) goto fail; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) goto fail; - i915_gem_object_pin_pages(obj); - sg = obj->pages; + sg = obj->mm.pages; bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size); - obj->dirty = 1; /* Backing store is now out of date */ + obj->mm.dirty = true; /* Backing store is now out of date */ i915_gem_object_unpin_pages(obj); if (WARN_ON(bytes != size)) { @@ -5091,13 +5076,13 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj, unsigned int n, unsigned int *offset) { - struct i915_gem_object_page_iter *iter = &obj->get_page; + struct i915_gem_object_page_iter *iter = &obj->mm.get_page; struct scatterlist *sg; unsigned int idx, count; might_sleep(); GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); - GEM_BUG_ON(obj->pages_pin_count == 0); + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); /* As we iterate forward through the sg, we record each entry in a * radixtree for quick repeated (backwards) lookups. If we have seen @@ -5222,7 +5207,7 @@ i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, struct page *page; page = i915_gem_object_get_page(obj, n); - if (!obj->dirty) + if (!obj->mm.dirty) set_page_dirty(page); return page; diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c index aa4e1e043b4e..e0f38e5c0fbb 100644 --- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c +++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c @@ -130,11 +130,10 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, return obj; } - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ERR_PTR(ret); list_move_tail(&obj->batch_pool_link, list); - i915_gem_object_pin_pages(obj); return obj; } diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c index 97c9d68b45df..10441dc72e73 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c @@ -48,12 +48,10 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme if (ret) goto err; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) goto err_unlock; - i915_gem_object_pin_pages(obj); - /* Copy sg so that we make an independent mapping */ st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (st == NULL) { @@ -61,13 +59,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme goto err_unpin; } - ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL); + ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL); if (ret) goto err_free; - src = obj->pages->sgl; + src = obj->mm.pages->sgl; dst = st->sgl; - for (i = 0; i < obj->pages->nents; i++) { + for (i = 0; i < obj->mm.pages->nents; i++) { sg_set_page(dst, sg_page(src), src->length, 0); dst = sg_next(dst); src = sg_next(src); @@ -299,14 +297,14 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) if (IS_ERR(sg)) return PTR_ERR(sg); - obj->pages = sg; + obj->mm.pages = sg; return 0; } static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj) { dma_buf_unmap_attachment(obj->base.import_attach, - obj->pages, DMA_BIDIRECTIONAL); + obj->mm.pages, DMA_BIDIRECTIONAL); } static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = { diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 4cafce97998a..d95c4e02eeb9 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -1281,7 +1281,7 @@ void i915_vma_move_to_active(struct i915_vma *vma, GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - obj->dirty = 1; /* be paranoid */ + obj->mm.dirty = true; /* be paranoid */ /* Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c index 3c5a8082cac3..5aadab59f071 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence.c +++ b/drivers/gpu/drm/i915/i915_gem_fence.c @@ -664,7 +664,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) return; i = 0; - for_each_sgt_page(page, sgt_iter, obj->pages) { + for_each_sgt_page(page, sgt_iter, obj->mm.pages) { char new_bit_17 = page_to_phys(page) >> 17; if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) { @@ -703,7 +703,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i = 0; - for_each_sgt_page(page, sgt_iter, obj->pages) { + for_each_sgt_page(page, sgt_iter, obj->mm.pages) { if (page_to_phys(page) & (1 << 17)) __set_bit(i, obj->bit_17); else diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index b3f341fe77bf..794ccc4cffaa 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -175,7 +175,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma, { u32 pte_flags = 0; - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages; /* Currently applicable only to VLV */ if (vma->obj->gt_ro) @@ -2373,7 +2373,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev) int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) { if (!dma_map_sg(&obj->base.dev->pdev->dev, - obj->pages->sgl, obj->pages->nents, + obj->mm.pages->sgl, obj->mm.pages->nents, PCI_DMA_BIDIRECTIONAL)) return -ENOSPC; @@ -2710,7 +2710,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } - dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents, + dma_unmap_sg(kdev, obj->mm.pages->sgl, obj->mm.pages->nents, PCI_DMA_BIDIRECTIONAL); } @@ -3548,7 +3548,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info, /* Populate source page list from the object. */ i = 0; - for_each_sgt_dma(dma_addr, sgt_iter, obj->pages) + for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages) page_addr_list[i++] = dma_addr; GEM_BUG_ON(i != n_pages); @@ -3641,7 +3641,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma) return 0; if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages; else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) vma->pages = intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj); diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c index 02e66fa170b0..08a2576ff7d2 100644 --- a/drivers/gpu/drm/i915/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/i915_gem_internal.c @@ -102,10 +102,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) sg = __sg_next(sg); } while (1); - obj->pages = st; + obj->mm.pages = st; if (i915_gem_gtt_prepare_object(obj)) { - obj->pages = NULL; + obj->mm.pages = NULL; goto err; } @@ -114,7 +114,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) * and the caller is expected to repopulate - the contents of this * object are only valid whilst active and pinned. */ - obj->madv = I915_MADV_DONTNEED; + obj->mm.madv = I915_MADV_DONTNEED; return 0; err: @@ -126,10 +126,10 @@ err: static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj) { i915_gem_gtt_finish_object(obj); - internal_free_pages(obj->pages); + internal_free_pages(obj->mm.pages); - obj->dirty = 0; - obj->madv = I915_MADV_WILLNEED; + obj->mm.dirty = false; + obj->mm.madv = I915_MADV_WILLNEED; } static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c index 05293246e0b9..57918f21ecc3 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.c +++ b/drivers/gpu/drm/i915/i915_gem_render_state.c @@ -230,7 +230,7 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req) return 0; /* Recreate the page after shrinking */ - if (!so->vma->obj->pages) + if (!so->vma->obj->mm.pages) so->batch_offset = -1; ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH); diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index de25b6e0a101..124f69a80162 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) * to the GPU, simply unbinding from the GPU is not going to succeed * in releasing our pin count on the pages themselves. */ - if (obj->pages_pin_count > obj->bind_count) + if (obj->mm.pages_pin_count > obj->bind_count) return false; if (any_vma_pinned(obj)) @@ -88,7 +88,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) * discard the contents (because the user has marked them as being * purgeable) or if we can move their contents out to swap. */ - return swap_available() || obj->madv == I915_MADV_DONTNEED; + return swap_available() || obj->mm.madv == I915_MADV_DONTNEED; } /** @@ -175,11 +175,11 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, list_move_tail(&obj->global_list, &still_in_list); if (flags & I915_SHRINK_PURGEABLE && - obj->madv != I915_MADV_DONTNEED) + obj->mm.madv != I915_MADV_DONTNEED) continue; if (flags & I915_SHRINK_VMAPS && - !is_vmalloc_addr(obj->mapping)) + !is_vmalloc_addr(obj->mm.mapping)) continue; if (!(flags & I915_SHRINK_ACTIVE) && @@ -194,7 +194,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, /* For the unbound phase, this should be a no-op! */ i915_gem_object_unbind(obj); - if (i915_gem_object_put_pages(obj) == 0) + if (__i915_gem_object_put_pages(obj) == 0) count += obj->base.size >> PAGE_SHIFT; i915_gem_object_put(obj); diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index 70e61bc35c60..0acbdcbd8f3a 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -555,16 +555,17 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj) { /* Should only be called during free */ - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(obj->mm.pages); + kfree(obj->mm.pages); } - static void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); + __i915_gem_object_unpin_pages(obj); + if (obj->stolen) { i915_gem_stolen_remove_node(dev_priv, obj->stolen); kfree(obj->stolen); @@ -590,15 +591,16 @@ _i915_gem_object_create_stolen(struct drm_device *dev, drm_gem_private_object_init(dev, &obj->base, stolen->size); i915_gem_object_init(obj, &i915_gem_object_stolen_ops); - obj->pages = i915_pages_create_for_stolen(dev, - stolen->start, stolen->size); - if (obj->pages == NULL) + obj->mm.pages = i915_pages_create_for_stolen(dev, + stolen->start, + stolen->size); + if (!obj->mm.pages) goto cleanup; - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx = 0; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx = 0; - i915_gem_object_pin_pages(obj); + __i915_gem_object_pin_pages(obj); obj->stolen = stolen; obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; @@ -718,14 +720,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, goto err; } - vma->pages = obj->pages; + vma->pages = obj->mm.pages; vma->flags |= I915_VMA_GLOBAL_BIND; __i915_vma_set_map_and_fenceable(vma); list_move_tail(&vma->vm_link, &ggtt->base.inactive_list); obj->bind_count++; list_add_tail(&obj->global_list, &dev_priv->mm.bound_list); - i915_gem_object_pin_pages(obj); + __i915_gem_object_pin_pages(obj); return obj; diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index 71f80d2a487c..34d5ada49720 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -259,13 +259,13 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, if (!err) { struct i915_vma *vma; - if (obj->pages && - obj->madv == I915_MADV_WILLNEED && + if (obj->mm.pages && + obj->mm.madv == I915_MADV_WILLNEED && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { if (args->tiling_mode == I915_TILING_NONE) - i915_gem_object_unpin_pages(obj); + __i915_gem_object_unpin_pages(obj); if (!i915_gem_object_is_tiled(obj)) - i915_gem_object_pin_pages(obj); + __i915_gem_object_pin_pages(obj); } list_for_each_entry(vma, &obj->vma_list, obj_link) { diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index e2fa970bb629..0cbc8f70317e 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -73,10 +73,10 @@ static void cancel_userptr(struct work_struct *work) /* Cancel any active worker and force us to re-evaluate gup */ obj->userptr.work = NULL; - if (obj->pages != NULL) { + if (obj->mm.pages) { /* We are inside a kthread context and can't be interrupted */ WARN_ON(i915_gem_object_unbind(obj)); - WARN_ON(i915_gem_object_put_pages(obj)); + WARN_ON(__i915_gem_object_put_pages(obj)); } i915_gem_object_put(obj); @@ -432,15 +432,15 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, { int ret; - ret = st_set_pages(&obj->pages, pvec, num_pages); + ret = st_set_pages(&obj->mm.pages, pvec, num_pages); if (ret) return ret; ret = i915_gem_gtt_prepare_object(obj); if (ret) { - sg_free_table(obj->pages); - kfree(obj->pages); - obj->pages = NULL; + sg_free_table(obj->mm.pages); + kfree(obj->mm.pages); + obj->mm.pages = NULL; } return ret; @@ -530,8 +530,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) if (ret == 0) { list_add_tail(&obj->global_list, &to_i915(dev)->mm.unbound_list); - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx = 0; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx = 0; pinned = 0; } } @@ -672,22 +672,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) BUG_ON(obj->userptr.work != NULL); __i915_gem_userptr_set_active(obj, false); - if (obj->madv != I915_MADV_WILLNEED) - obj->dirty = 0; + if (obj->mm.madv != I915_MADV_WILLNEED) + obj->mm.dirty = false; i915_gem_gtt_finish_object(obj); - for_each_sgt_page(page, sgt_iter, obj->pages) { - if (obj->dirty) + for_each_sgt_page(page, sgt_iter, obj->mm.pages) { + if (obj->mm.dirty) set_page_dirty(page); mark_page_accessed(page); put_page(page); } - obj->dirty = 0; + obj->mm.dirty = false; - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(obj->mm.pages); + kfree(obj->mm.pages); } static void diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index d5feace8de0d..5bbb37209aa5 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -896,8 +896,8 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->write_domain = obj->base.write_domain; err->fence_reg = vma->fence ? vma->fence->id : -1; err->tiling = i915_gem_object_get_tiling(obj); - err->dirty = obj->dirty; - err->purgeable = obj->madv != I915_MADV_WILLNEED; + err->dirty = obj->mm.dirty; + err->purgeable = obj->mm.madv != I915_MADV_WILLNEED; err->userptr = obj->userptr.mm != NULL; err->cache_level = obj->cache_level; } diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 1c1bd30e8b2d..cb30549dfd40 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -744,7 +744,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx, ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = i915_ggtt_offset(ce->ring->vma); - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true; /* Invalidate GuC TLB. */ if (i915.enable_guc_submission) { @@ -2042,7 +2042,7 @@ populate_lr_context(struct i915_gem_context *ctx, DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret); return ret; } - ctx_obj->dirty = true; + ctx_obj->mm.dirty = true; /* The second page of the context object contains some fields which must * be set up prior to the first execution. */ @@ -2180,7 +2180,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv) reg[CTX_RING_HEAD+1] = 0; reg[CTX_RING_TAIL+1] = 0; - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true; i915_gem_object_unpin_map(ce->state->obj); ce->ring->head = ce->ring->tail = 0; -- cgit v1.2.3 From fbbd37b36fa5e16a03aca07a922192d3db28588d Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:42 +0100 Subject: drm/i915: Move object release to a freelist + worker We want to hide the latency of releasing objects and their backing storage from the submission, so we move the actual free to a worker. This allows us to switch to struct_mutex freeing of the object in the next patch. Furthermore, if we know that the object we are dereferencing remains valid for the duration of our access, we can forgo the usual synchronisation barriers and atomic reference counting. To ensure this we defer freeing an object til after an RCU grace period, such that any lookup of the object within an RCU read critical section will remain valid until after we exit that critical section. We also employ this delay for rate-limiting the serialisation on reallocation - we have to slow down object creation in order to prevent resource starvation (in particular, files). v2: Return early in i915_gem_tiling() ioctl to skip over superfluous work on error. Signed-off-by: Chris Wilson Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-19-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 15 ++- drivers/gpu/drm/i915/i915_drv.c | 19 ++-- drivers/gpu/drm/i915/i915_drv.h | 44 +++++++- drivers/gpu/drm/i915/i915_gem.c | 166 +++++++++++++++++++++---------- drivers/gpu/drm/i915/i915_gem_shrinker.c | 14 ++- drivers/gpu/drm/i915/i915_gem_tiling.c | 21 ++-- 6 files changed, 202 insertions(+), 77 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index e97a16cb6b67..b0b01002c0d1 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -4957,10 +4957,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops, #define DROP_BOUND 0x2 #define DROP_RETIRE 0x4 #define DROP_ACTIVE 0x8 -#define DROP_ALL (DROP_UNBOUND | \ - DROP_BOUND | \ - DROP_RETIRE | \ - DROP_ACTIVE) +#define DROP_FREED 0x10 +#define DROP_ALL (DROP_UNBOUND | \ + DROP_BOUND | \ + DROP_RETIRE | \ + DROP_ACTIVE | \ + DROP_FREED) static int i915_drop_caches_get(void *data, u64 *val) { @@ -5004,6 +5006,11 @@ i915_drop_caches_set(void *data, u64 val) unlock: mutex_unlock(&dev->struct_mutex); + if (val & DROP_FREED) { + synchronize_rcu(); + flush_work(&dev_priv->mm.free_work); + } + return ret; } diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index af3559d34328..b308f7173b6d 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -537,14 +537,17 @@ static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { .can_switch = i915_switcheroo_can_switch, }; -static void i915_gem_fini(struct drm_device *dev) +static void i915_gem_fini(struct drm_i915_private *dev_priv) { - mutex_lock(&dev->struct_mutex); - i915_gem_cleanup_engines(dev); - i915_gem_context_fini(dev); - mutex_unlock(&dev->struct_mutex); + mutex_lock(&dev_priv->drm.struct_mutex); + i915_gem_cleanup_engines(&dev_priv->drm); + i915_gem_context_fini(&dev_priv->drm); + mutex_unlock(&dev_priv->drm.struct_mutex); + + synchronize_rcu(); + flush_work(&dev_priv->mm.free_work); - WARN_ON(!list_empty(&to_i915(dev)->context_list)); + WARN_ON(!list_empty(&dev_priv->context_list)); } static int i915_load_modeset_init(struct drm_device *dev) @@ -619,7 +622,7 @@ static int i915_load_modeset_init(struct drm_device *dev) cleanup_gem: if (i915_gem_suspend(dev)) DRM_ERROR("failed to idle hardware; continuing to unload!\n"); - i915_gem_fini(dev); + i915_gem_fini(dev_priv); cleanup_irq: intel_guc_fini(dev); drm_irq_uninstall(dev); @@ -1305,7 +1308,7 @@ void i915_driver_unload(struct drm_device *dev) drain_workqueue(dev_priv->wq); intel_guc_fini(dev); - i915_gem_fini(dev); + i915_gem_fini(dev_priv); intel_fbc_cleanup_cfb(dev_priv); intel_power_domains_fini(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index f69e0e03e259..6f648a3b96f4 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -1365,8 +1365,8 @@ struct i915_gem_mm { struct list_head bound_list; /** * List of objects which are not bound to the GTT (thus - * are idle and not used by the GPU) but still have - * (presumably uncached) pages still attached. + * are idle and not used by the GPU). These objects may or may + * not actually have any pages attached. */ struct list_head unbound_list; @@ -1375,6 +1375,12 @@ struct i915_gem_mm { */ struct list_head userfault_list; + /** + * List of objects which are pending destruction. + */ + struct llist_head free_list; + struct work_struct free_work; + /** Usable portion of the GTT for GEM */ unsigned long stolen_base; /* limited to low memory (32-bit) */ @@ -2224,6 +2230,10 @@ struct drm_i915_gem_object { /** Stolen memory for this object, instead of being backed by shmem. */ struct drm_mm_node *stolen; struct list_head global_list; + union { + struct rcu_head rcu; + struct llist_node freed; + }; /** * Whether the object is currently in the GGTT mmap. @@ -2341,10 +2351,38 @@ to_intel_bo(struct drm_gem_object *gem) return container_of(gem, struct drm_i915_gem_object, base); } +/** + * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle + * @filp: DRM file private date + * @handle: userspace handle + * + * Returns: + * + * A pointer to the object named by the handle if such exists on @filp, NULL + * otherwise. This object is only valid whilst under the RCU read lock, and + * note carefully the object may be in the process of being destroyed. + */ +static inline struct drm_i915_gem_object * +i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) +{ +#ifdef CONFIG_LOCKDEP + WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); +#endif + return idr_find(&file->object_idr, handle); +} + static inline struct drm_i915_gem_object * i915_gem_object_lookup(struct drm_file *file, u32 handle) { - return to_intel_bo(drm_gem_object_lookup(file, handle)); + struct drm_i915_gem_object *obj; + + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, handle); + if (obj && !kref_get_unless_zero(&obj->base.refcount)) + obj = NULL; + rcu_read_unlock(); + + return obj; } __deprecated diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 9f1bb1f80787..07b2eb6bc572 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -42,6 +42,7 @@ #include #include +static void i915_gem_flush_free_objects(struct drm_i915_private *i915); static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); @@ -648,6 +649,8 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_create *args = data; + i915_gem_flush_free_objects(to_i915(dev)); + return i915_gem_create(file, dev, args->size, &args->handle); } @@ -3524,10 +3527,14 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; + int err = 0; - obj = i915_gem_object_lookup(file, args->handle); - if (!obj) - return -ENOENT; + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); + if (!obj) { + err = -ENOENT; + goto out; + } switch (obj->cache_level) { case I915_CACHE_LLC: @@ -3543,9 +3550,9 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, args->caching = I915_CACHING_NONE; break; } - - i915_gem_object_put_unlocked(obj); - return 0; +out: + rcu_read_unlock(); + return err; } int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, @@ -4089,10 +4096,14 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_busy *args = data; struct drm_i915_gem_object *obj; unsigned long active; + int err; - obj = i915_gem_object_lookup(file, args->handle); - if (!obj) - return -ENOENT; + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); + if (!obj) { + err = -ENOENT; + goto out; + } args->busy = 0; active = __I915_BO_ACTIVE(obj); @@ -4122,7 +4133,6 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, * are busy is not completely reliable - we only guarantee * that the object was busy. */ - rcu_read_lock(); for_each_active(active, idx) args->busy |= busy_check_reader(&obj->last_read[idx]); @@ -4140,12 +4150,11 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, * the result. */ args->busy |= busy_check_writer(&obj->last_write); - - rcu_read_unlock(); } - i915_gem_object_put_unlocked(obj); - return 0; +out: + rcu_read_unlock(); + return err; } int @@ -4308,7 +4317,6 @@ i915_gem_object_create(struct drm_device *dev, u64 size) fail: i915_gem_object_free(obj); - return ERR_PTR(ret); } @@ -4336,16 +4344,69 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj) return atomic_long_read(&obj->base.filp->f_count) == 1; } -void i915_gem_free_object(struct drm_gem_object *gem_obj) +static void __i915_gem_free_objects(struct drm_i915_private *i915, + struct llist_node *freed) { - struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_vma *vma, *next; + struct drm_i915_gem_object *obj, *on; - intel_runtime_pm_get(dev_priv); + mutex_lock(&i915->drm.struct_mutex); + intel_runtime_pm_get(i915); + llist_for_each_entry(obj, freed, freed) { + struct i915_vma *vma, *vn; + + trace_i915_gem_object_destroy(obj); + + GEM_BUG_ON(i915_gem_object_is_active(obj)); + list_for_each_entry_safe(vma, vn, + &obj->vma_list, obj_link) { + GEM_BUG_ON(!i915_vma_is_ggtt(vma)); + GEM_BUG_ON(i915_vma_is_active(vma)); + vma->flags &= ~I915_VMA_PIN_MASK; + i915_vma_close(vma); + } + + list_del(&obj->global_list); + } + intel_runtime_pm_put(i915); + mutex_unlock(&i915->drm.struct_mutex); + + llist_for_each_entry_safe(obj, on, freed, freed) { + GEM_BUG_ON(obj->bind_count); + GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits)); + + if (obj->ops->release) + obj->ops->release(obj); + + if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) + atomic_set(&obj->mm.pages_pin_count, 0); + __i915_gem_object_put_pages(obj); + GEM_BUG_ON(obj->mm.pages); + + if (obj->base.import_attach) + drm_prime_gem_destroy(&obj->base, NULL); + + drm_gem_object_release(&obj->base); + i915_gem_info_remove_obj(i915, obj->base.size); + + kfree(obj->bit_17); + i915_gem_object_free(obj); + } +} - trace_i915_gem_object_destroy(obj); +static void i915_gem_flush_free_objects(struct drm_i915_private *i915) +{ + struct llist_node *freed; + + freed = llist_del_all(&i915->mm.free_list); + if (unlikely(freed)) + __i915_gem_free_objects(i915, freed); +} + +static void __i915_gem_free_work(struct work_struct *work) +{ + struct drm_i915_private *i915 = + container_of(work, struct drm_i915_private, mm.free_work); + struct llist_node *freed; /* All file-owned VMA should have been released by this point through * i915_gem_close_object(), or earlier by i915_gem_context_close(). @@ -4354,42 +4415,44 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) * the GTT either for the user or for scanout). Those VMA still need to * unbound now. */ - list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) { - GEM_BUG_ON(!i915_vma_is_ggtt(vma)); - GEM_BUG_ON(i915_vma_is_active(vma)); - vma->flags &= ~I915_VMA_PIN_MASK; - i915_vma_close(vma); - } - GEM_BUG_ON(obj->bind_count); - WARN_ON(atomic_read(&obj->frontbuffer_bits)); + while ((freed = llist_del_all(&i915->mm.free_list))) + __i915_gem_free_objects(i915, freed); +} - if (obj->mm.pages && obj->mm.madv == I915_MADV_WILLNEED && - dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES && - i915_gem_object_is_tiled(obj)) - __i915_gem_object_unpin_pages(obj); +static void __i915_gem_free_object_rcu(struct rcu_head *head) +{ + struct drm_i915_gem_object *obj = + container_of(head, typeof(*obj), rcu); + struct drm_i915_private *i915 = to_i915(obj->base.dev); - if (obj->ops->release) - obj->ops->release(obj); + /* We can't simply use call_rcu() from i915_gem_free_object() + * as we need to block whilst unbinding, and the call_rcu + * task may be called from softirq context. So we take a + * detour through a worker. + */ + if (llist_add(&obj->freed, &i915->mm.free_list)) + schedule_work(&i915->mm.free_work); +} + +void i915_gem_free_object(struct drm_gem_object *gem_obj) +{ + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) - atomic_set(&obj->mm.pages_pin_count, 0); if (discard_backing_storage(obj)) obj->mm.madv = I915_MADV_DONTNEED; - __i915_gem_object_put_pages(obj); - GEM_BUG_ON(obj->mm.pages); - - if (obj->base.import_attach) - drm_prime_gem_destroy(&obj->base, NULL); - - drm_gem_object_release(&obj->base); - i915_gem_info_remove_obj(dev_priv, obj->base.size); - - kfree(obj->bit_17); - i915_gem_object_free(obj); + if (obj->mm.pages && obj->mm.madv == I915_MADV_WILLNEED && + to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES && + i915_gem_object_is_tiled(obj)) + __i915_gem_object_unpin_pages(obj); - intel_runtime_pm_put(dev_priv); + /* Before we free the object, make sure any pure RCU-only + * read-side critical sections are complete, e.g. + * i915_gem_busy_ioctl(). For the corresponding synchronized + * lookup see i915_gem_object_lookup_rcu(). + */ + call_rcu(&obj->rcu, __i915_gem_free_object_rcu); } void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj) @@ -4438,6 +4501,7 @@ int i915_gem_suspend(struct drm_device *dev) cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); cancel_delayed_work_sync(&dev_priv->gt.retire_work); flush_delayed_work(&dev_priv->gt.idle_work); + flush_work(&dev_priv->mm.free_work); /* Assert that we sucessfully flushed all the work and * reset the GPU back to its idle, low power state. @@ -4753,6 +4817,8 @@ i915_gem_load_init(struct drm_device *dev) NULL); INIT_LIST_HEAD(&dev_priv->context_list); + INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work); + init_llist_head(&dev_priv->mm.free_list); INIT_LIST_HEAD(&dev_priv->mm.unbound_list); INIT_LIST_HEAD(&dev_priv->mm.bound_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index c8a4c40ec2c2..0241658af16b 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -201,6 +201,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, typeof(*obj), global_list))) { list_move_tail(&obj->global_list, &still_in_list); + if (!obj->mm.pages) { + list_del_init(&obj->global_list); + continue; + } if (flags & I915_SHRINK_PURGEABLE && obj->mm.madv != I915_MADV_DONTNEED) @@ -218,8 +222,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, if (!can_release_pages(obj)) continue; - i915_gem_object_get(obj); - if (unsafe_drop_pages(obj)) { mutex_lock(&obj->mm.lock); if (!obj->mm.pages) { @@ -228,8 +230,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, } mutex_unlock(&obj->mm.lock); } - - i915_gem_object_put(obj); } list_splice(&still_in_list, phase->list); } @@ -396,12 +396,18 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) */ unbound = bound = unevictable = 0; list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + if (!obj->mm.pages) + continue; + if (!can_release_pages(obj)) unevictable += obj->base.size >> PAGE_SHIFT; else unbound += obj->base.size >> PAGE_SHIFT; } list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + if (!obj->mm.pages) + continue; + if (!can_release_pages(obj)) unevictable += obj->base.size >> PAGE_SHIFT; else diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index 6608799ee1f9..c63a9cf4da33 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -325,12 +325,19 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_i915_gem_get_tiling *args = data; struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; + int err = -ENOENT; + + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); + if (obj) { + args->tiling_mode = + READ_ONCE(obj->tiling_and_stride) & TILING_MASK; + err = 0; + } + rcu_read_unlock(); + if (unlikely(err)) + return err; - obj = i915_gem_object_lookup(file, args->handle); - if (!obj) - return -ENOENT; - - args->tiling_mode = READ_ONCE(obj->tiling_and_stride) & TILING_MASK; switch (args->tiling_mode) { case I915_TILING_X: args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; @@ -338,11 +345,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, case I915_TILING_Y: args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; break; + default: case I915_TILING_NONE: args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; break; - default: - DRM_ERROR("unknown tiling mode\n"); } /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ @@ -355,6 +361,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; - i915_gem_object_put_unlocked(obj); return 0; } -- cgit v1.2.3 From d07f0e59b2c762584478920cd2d11fba2980a94a Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:44 +0100 Subject: drm/i915: Move GEM activity tracking into a common struct reservation_object In preparation to support many distinct timelines, we need to expand the activity tracking on the GEM object to handle more than just a request per engine. We already use the struct reservation_object on the dma-buf to handle many fence contexts, so integrating that into the GEM object itself is the preferred solution. (For example, we can now share the same reservation_object between every consumer/producer using this buffer and skip the manual import/export via dma-buf.) v2: Reimplement busy-ioctl (by walking the reservation object), postpone the ABI change for another day. Similarly use the reservation object to find the last_write request (if active and from i915) for choosing display CS flips. Caveats: * busy-ioctl: busy-ioctl only reports on the native fences, it will not warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is being rendered to by external fences. It also will not report the same busy state as wait-ioctl (or polling on the dma-buf) in the same circumstances. On the plus side, it does retain reporting of which *i915* engines are engaged with this object. * non-blocking atomic modesets take a step backwards as the wait for render completion blocks the ioctl. This is fixed in a subsequent patch to use a fence instead for awaiting on the rendering, see "drm/i915: Restore nonblocking awaits for modesetting" * dynamic array manipulation for shared-fences in reservation is slower than the previous lockless static assignment (e.g. gem_exec_lut_handle runtime on ivb goes from 42s to 66s), mainly due to atomic operations (maintaining the fence refcounts). * loss of object-level retirement callbacks, emulated by VMA retirement tracking. * minor loss of object-level last activity information from debugfs, could be replaced with per-vma information if desired Signed-off-by: Chris Wilson Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 15 +- drivers/gpu/drm/i915/i915_drv.h | 62 +++---- drivers/gpu/drm/i915/i915_gem.c | 263 ++++++++--------------------- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 11 +- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 53 +----- drivers/gpu/drm/i915/i915_gem_dmabuf.h | 45 ----- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 55 ++---- drivers/gpu/drm/i915/i915_gem_gtt.c | 32 ++++ drivers/gpu/drm/i915/i915_gem_gtt.h | 1 + drivers/gpu/drm/i915/i915_gem_request.c | 48 +++--- drivers/gpu/drm/i915/i915_gem_request.h | 35 +--- drivers/gpu/drm/i915/i915_gpu_error.c | 6 +- drivers/gpu/drm/i915/intel_atomic_plane.c | 2 - drivers/gpu/drm/i915/intel_display.c | 114 +++---------- drivers/gpu/drm/i915/intel_drv.h | 3 - 15 files changed, 212 insertions(+), 533 deletions(-) delete mode 100644 drivers/gpu/drm/i915/i915_gem_dmabuf.h (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index b0b01002c0d1..b6325065c8e8 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -136,11 +136,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) struct i915_vma *vma; unsigned int frontbuffer_bits; int pin_count = 0; - enum intel_engine_id id; lockdep_assert_held(&obj->base.dev->struct_mutex); - seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x [ ", + seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x %s%s%s", &obj->base, get_active_flag(obj), get_pin_flag(obj), @@ -149,14 +148,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) get_pin_mapped_flag(obj), obj->base.size / 1024, obj->base.read_domains, - obj->base.write_domain); - for_each_engine(engine, dev_priv, id) - seq_printf(m, "%x ", - i915_gem_active_get_seqno(&obj->last_read[id], - &obj->base.dev->struct_mutex)); - seq_printf(m, "] %x %s%s%s", - i915_gem_active_get_seqno(&obj->last_write, - &obj->base.dev->struct_mutex), + obj->base.write_domain, i915_cache_level_str(dev_priv, obj->cache_level), obj->mm.dirty ? " dirty" : "", obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : ""); @@ -187,8 +179,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) if (obj->stolen) seq_printf(m, " (stolen: %08llx)", obj->stolen->start); - engine = i915_gem_active_get_engine(&obj->last_write, - &dev_priv->drm.struct_mutex); + engine = i915_gem_object_last_write_engine(obj); if (engine) seq_printf(m, " (%s)", engine->name); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index c3c49bc4d2ac..693ee69a4715 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -2246,21 +2247,12 @@ struct drm_i915_gem_object { struct list_head batch_pool_link; unsigned long flags; - /** - * This is set if the object is on the active lists (has pending - * rendering and so a non-zero seqno), and is not set if it i s on - * inactive (ready to be unbound) list. - */ -#define I915_BO_ACTIVE_SHIFT 0 -#define I915_BO_ACTIVE_MASK ((1 << I915_NUM_ENGINES) - 1) -#define __I915_BO_ACTIVE(bo) \ - ((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK) /** * Have we taken a reference for the object for incomplete GPU * activity? */ -#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES) +#define I915_BO_ACTIVE_REF 0 /* * Is the object to be mapped as read-only to the GPU @@ -2281,6 +2273,7 @@ struct drm_i915_gem_object { /** Count of VMA actually bound by this object */ unsigned int bind_count; + unsigned int active_count; unsigned int pin_display; struct { @@ -2320,8 +2313,7 @@ struct drm_i915_gem_object { * read request. This allows for the CPU to read from an active * buffer by only waiting for the write to complete. */ - struct i915_gem_active last_read[I915_NUM_ENGINES]; - struct i915_gem_active last_write; + struct reservation_object *resv; /** References from framebuffers, locks out tiling changes. */ unsigned long framebuffer_references; @@ -2340,6 +2332,8 @@ struct drm_i915_gem_object { /** for phys allocated objects */ struct drm_dma_handle *phys_handle; + + struct reservation_object __builtin_resv; }; static inline struct drm_i915_gem_object * @@ -2425,35 +2419,10 @@ i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; } -static inline unsigned long -i915_gem_object_get_active(const struct drm_i915_gem_object *obj) -{ - return (obj->flags >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK; -} - static inline bool i915_gem_object_is_active(const struct drm_i915_gem_object *obj) { - return i915_gem_object_get_active(obj); -} - -static inline void -i915_gem_object_set_active(struct drm_i915_gem_object *obj, int engine) -{ - obj->flags |= BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline void -i915_gem_object_clear_active(struct drm_i915_gem_object *obj, int engine) -{ - obj->flags &= ~BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline bool -i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj, - int engine) -{ - return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT); + return obj->active_count; } static inline bool @@ -2496,6 +2465,23 @@ i915_gem_object_get_stride(struct drm_i915_gem_object *obj) return obj->tiling_and_stride & STRIDE_MASK; } +static inline struct intel_engine_cs * +i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj) +{ + struct intel_engine_cs *engine = NULL; + struct dma_fence *fence; + + rcu_read_lock(); + fence = reservation_object_get_excl_rcu(obj->resv); + rcu_read_unlock(); + + if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence)) + engine = to_request(fence)->engine; + dma_fence_put(fence); + + return engine; +} + static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) { i915_gem_object_get(vma->obj); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index fdcbaab21fdd..1161a21ec810 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -29,7 +29,6 @@ #include #include #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "i915_vgpu.h" #include "i915_trace.h" #include "intel_drv.h" @@ -447,11 +446,6 @@ i915_gem_object_wait(struct drm_i915_gem_object *obj, long timeout, struct intel_rps_client *rps) { - struct reservation_object *resv; - struct i915_gem_active *active; - unsigned long active_mask; - int idx; - might_sleep(); #if IS_ENABLED(CONFIG_LOCKDEP) GEM_BUG_ON(debug_locks && @@ -460,33 +454,9 @@ i915_gem_object_wait(struct drm_i915_gem_object *obj, #endif GEM_BUG_ON(timeout < 0); - if (flags & I915_WAIT_ALL) { - active = obj->last_read; - active_mask = i915_gem_object_get_active(obj); - } else { - active_mask = 1; - active = &obj->last_write; - } - - for_each_active(active_mask, idx) { - struct drm_i915_gem_request *request; - - request = i915_gem_active_get_unlocked(&active[idx]); - if (request) { - timeout = i915_gem_object_wait_fence(&request->fence, - flags, timeout, - rps); - i915_gem_request_put(request); - } - if (timeout < 0) - return timeout; - } - - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) - timeout = i915_gem_object_wait_reservation(resv, - flags, timeout, - rps); + timeout = i915_gem_object_wait_reservation(obj->resv, + flags, timeout, + rps); return timeout < 0 ? timeout : 0; } @@ -2549,44 +2519,6 @@ err_unlock: goto out_unlock; } -static void -i915_gem_object_retire__write(struct i915_gem_active *active, - struct drm_i915_gem_request *request) -{ - struct drm_i915_gem_object *obj = - container_of(active, struct drm_i915_gem_object, last_write); - - intel_fb_obj_flush(obj, true, ORIGIN_CS); -} - -static void -i915_gem_object_retire__read(struct i915_gem_active *active, - struct drm_i915_gem_request *request) -{ - int idx = request->engine->id; - struct drm_i915_gem_object *obj = - container_of(active, struct drm_i915_gem_object, last_read[idx]); - - GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx)); - - i915_gem_object_clear_active(obj, idx); - if (i915_gem_object_is_active(obj)) - return; - - /* Bump our place on the bound list to keep it roughly in LRU order - * so that we don't steal from recently used but inactive objects - * (unless we are forced to ofc!) - */ - if (obj->bind_count) - list_move_tail(&obj->global_list, - &request->i915->mm.bound_list); - - if (i915_gem_object_has_active_reference(obj)) { - i915_gem_object_clear_active_reference(obj); - i915_gem_object_put(obj); - } -} - static bool i915_context_is_banned(const struct i915_gem_context *ctx) { unsigned long elapsed; @@ -2966,6 +2898,13 @@ int i915_vma_unbind(struct i915_vma *vma) * In order to prevent it from being recursively closed, * take a pin on the vma so that the second unbind is * aborted. + * + * Even more scary is that the retire callback may free + * the object (last active vma). To prevent the explosion + * we defer the actual object free to a worker that can + * only proceed once it acquires the struct_mutex (which + * we currently hold, therefore it cannot free this object + * before we are finished). */ __i915_vma_pin(vma); @@ -4010,83 +3949,42 @@ static __always_inline unsigned int __busy_write_id(unsigned int id) } static __always_inline unsigned int -__busy_set_if_active(const struct i915_gem_active *active, +__busy_set_if_active(const struct dma_fence *fence, unsigned int (*flag)(unsigned int id)) { - struct drm_i915_gem_request *request; - - request = rcu_dereference(active->request); - if (!request || i915_gem_request_completed(request)) - return 0; + struct drm_i915_gem_request *rq; - /* This is racy. See __i915_gem_active_get_rcu() for an in detail - * discussion of how to handle the race correctly, but for reporting - * the busy state we err on the side of potentially reporting the - * wrong engine as being busy (but we guarantee that the result - * is at least self-consistent). - * - * As we use SLAB_DESTROY_BY_RCU, the request may be reallocated - * whilst we are inspecting it, even under the RCU read lock as we are. - * This means that there is a small window for the engine and/or the - * seqno to have been overwritten. The seqno will always be in the - * future compared to the intended, and so we know that if that - * seqno is idle (on whatever engine) our request is idle and the - * return 0 above is correct. + /* We have to check the current hw status of the fence as the uABI + * guarantees forward progress. We could rely on the idle worker + * to eventually flush us, but to minimise latency just ask the + * hardware. * - * The issue is that if the engine is switched, it is just as likely - * to report that it is busy (but since the switch happened, we know - * the request should be idle). So there is a small chance that a busy - * result is actually the wrong engine. - * - * So why don't we care? - * - * For starters, the busy ioctl is a heuristic that is by definition - * racy. Even with perfect serialisation in the driver, the hardware - * state is constantly advancing - the state we report to the user - * is stale. - * - * The critical information for the busy-ioctl is whether the object - * is idle as userspace relies on that to detect whether its next - * access will stall, or if it has missed submitting commands to - * the hardware allowing the GPU to stall. We never generate a - * false-positive for idleness, thus busy-ioctl is reliable at the - * most fundamental level, and we maintain the guarantee that a - * busy object left to itself will eventually become idle (and stay - * idle!). - * - * We allow ourselves the leeway of potentially misreporting the busy - * state because that is an optimisation heuristic that is constantly - * in flux. Being quickly able to detect the busy/idle state is much - * more important than accurate logging of exactly which engines were - * busy. - * - * For accuracy in reporting the engine, we could use - * - * result = 0; - * request = __i915_gem_active_get_rcu(active); - * if (request) { - * if (!i915_gem_request_completed(request)) - * result = flag(request->engine->exec_id); - * i915_gem_request_put(request); - * } - * - * but that still remains susceptible to both hardware and userspace - * races. So we accept making the result of that race slightly worse, - * given the rarity of the race and its low impact on the result. + * Note we only report on the status of native fences. */ - return flag(READ_ONCE(request->engine->exec_id)); + if (!dma_fence_is_i915(fence)) + return 0; + + /* opencode to_request() in order to avoid const warnings */ + rq = container_of(fence, struct drm_i915_gem_request, fence); + if (i915_gem_request_completed(rq)) + return 0; + + return flag(rq->engine->exec_id); } static __always_inline unsigned int -busy_check_reader(const struct i915_gem_active *active) +busy_check_reader(const struct dma_fence *fence) { - return __busy_set_if_active(active, __busy_read_flag); + return __busy_set_if_active(fence, __busy_read_flag); } static __always_inline unsigned int -busy_check_writer(const struct i915_gem_active *active) +busy_check_writer(const struct dma_fence *fence) { - return __busy_set_if_active(active, __busy_write_id); + if (!fence) + return 0; + + return __busy_set_if_active(fence, __busy_write_id); } int @@ -4095,63 +3993,55 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_busy *args = data; struct drm_i915_gem_object *obj; - unsigned long active; + struct reservation_object_list *list; + unsigned int seq; int err; + err = -ENOENT; rcu_read_lock(); obj = i915_gem_object_lookup_rcu(file, args->handle); - if (!obj) { - err = -ENOENT; + if (!obj) goto out; - } - args->busy = 0; - active = __I915_BO_ACTIVE(obj); - if (active) { - int idx; + /* A discrepancy here is that we do not report the status of + * non-i915 fences, i.e. even though we may report the object as idle, + * a call to set-domain may still stall waiting for foreign rendering. + * This also means that wait-ioctl may report an object as busy, + * where busy-ioctl considers it idle. + * + * We trade the ability to warn of foreign fences to report on which + * i915 engines are active for the object. + * + * Alternatively, we can trade that extra information on read/write + * activity with + * args->busy = + * !reservation_object_test_signaled_rcu(obj->resv, true); + * to report the overall busyness. This is what the wait-ioctl does. + * + */ +retry: + seq = raw_read_seqcount(&obj->resv->seq); - /* Yes, the lookups are intentionally racy. - * - * First, we cannot simply rely on __I915_BO_ACTIVE. We have - * to regard the value as stale and as our ABI guarantees - * forward progress, we confirm the status of each active - * request with the hardware. - * - * Even though we guard the pointer lookup by RCU, that only - * guarantees that the pointer and its contents remain - * dereferencable and does *not* mean that the request we - * have is the same as the one being tracked by the object. - * - * Consider that we lookup the request just as it is being - * retired and freed. We take a local copy of the pointer, - * but before we add its engine into the busy set, the other - * thread reallocates it and assigns it to a task on another - * engine with a fresh and incomplete seqno. Guarding against - * that requires careful serialisation and reference counting, - * i.e. using __i915_gem_active_get_request_rcu(). We don't, - * instead we expect that if the result is busy, which engines - * are busy is not completely reliable - we only guarantee - * that the object was busy. - */ + /* Translate the exclusive fence to the READ *and* WRITE engine */ + args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl)); - for_each_active(active, idx) - args->busy |= busy_check_reader(&obj->last_read[idx]); + /* Translate shared fences to READ set of engines */ + list = rcu_dereference(obj->resv->fence); + if (list) { + unsigned int shared_count = list->shared_count, i; - /* For ABI sanity, we only care that the write engine is in - * the set of read engines. This should be ensured by the - * ordering of setting last_read/last_write in - * i915_vma_move_to_active(), and then in reverse in retire. - * However, for good measure, we always report the last_write - * request as a busy read as well as being a busy write. - * - * We don't care that the set of active read/write engines - * may change during construction of the result, as it is - * equally liable to change before userspace can inspect - * the result. - */ - args->busy |= busy_check_writer(&obj->last_write); + for (i = 0; i < shared_count; ++i) { + struct dma_fence *fence = + rcu_dereference(list->shared[i]); + + args->busy |= busy_check_reader(fence); + } } + if (args->busy && read_seqcount_retry(&obj->resv->seq, seq)) + goto retry; + + err = 0; out: rcu_read_unlock(); return err; @@ -4216,23 +4106,19 @@ out: void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops) { - int i; - mutex_init(&obj->mm.lock); INIT_LIST_HEAD(&obj->global_list); INIT_LIST_HEAD(&obj->userfault_link); - for (i = 0; i < I915_NUM_ENGINES; i++) - init_request_active(&obj->last_read[i], - i915_gem_object_retire__read); - init_request_active(&obj->last_write, - i915_gem_object_retire__write); INIT_LIST_HEAD(&obj->obj_exec_link); INIT_LIST_HEAD(&obj->vma_list); INIT_LIST_HEAD(&obj->batch_pool_link); obj->ops = ops; + reservation_object_init(&obj->__builtin_resv); + obj->resv = &obj->__builtin_resv; + obj->frontbuffer_ggtt_origin = ORIGIN_GTT; obj->mm.madv = I915_MADV_WILLNEED; @@ -4385,6 +4271,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, if (obj->base.import_attach) drm_prime_gem_destroy(&obj->base, NULL); + reservation_object_fini(&obj->__builtin_resv); drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(i915, obj->base.size); diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c index e0f38e5c0fbb..b3bc119ec1bb 100644 --- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c +++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c @@ -114,11 +114,18 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, list_for_each_entry(tmp, list, batch_pool_link) { /* The batches are strictly LRU ordered */ - if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id], - &tmp->base.dev->struct_mutex)) + if (i915_gem_object_is_active(tmp)) break; + GEM_BUG_ON(!reservation_object_test_signaled_rcu(tmp->resv, + true)); + if (tmp->base.size >= size) { + /* Clear the set of shared fences early */ + ww_mutex_lock(&tmp->resv->lock, NULL); + reservation_object_add_excl_fence(tmp->resv, NULL); + ww_mutex_unlock(&tmp->resv->lock); + obj = tmp; break; } diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c index 4d45f20d11ed..5e38299b5df6 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c @@ -211,60 +211,17 @@ static const struct dma_buf_ops i915_dmabuf_ops = { .end_cpu_access = i915_gem_end_cpu_access, }; -static void export_fences(struct drm_i915_gem_object *obj, - struct dma_buf *dma_buf) -{ - struct reservation_object *resv = dma_buf->resv; - struct drm_i915_gem_request *req; - unsigned long active; - int idx; - - active = __I915_BO_ACTIVE(obj); - if (!active) - return; - - /* Serialise with execbuf to prevent concurrent fence-loops */ - mutex_lock(&obj->base.dev->struct_mutex); - - /* Mark the object for future fences before racily adding old fences */ - obj->base.dma_buf = dma_buf; - - ww_mutex_lock(&resv->lock, NULL); - - for_each_active(active, idx) { - req = i915_gem_active_get(&obj->last_read[idx], - &obj->base.dev->struct_mutex); - if (!req) - continue; - - if (reservation_object_reserve_shared(resv) == 0) - reservation_object_add_shared_fence(resv, &req->fence); - - i915_gem_request_put(req); - } - - req = i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); - if (req) { - reservation_object_add_excl_fence(resv, &req->fence); - i915_gem_request_put(req); - } - - ww_mutex_unlock(&resv->lock); - mutex_unlock(&obj->base.dev->struct_mutex); -} - struct dma_buf *i915_gem_prime_export(struct drm_device *dev, struct drm_gem_object *gem_obj, int flags) { struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - struct dma_buf *dma_buf; exp_info.ops = &i915_dmabuf_ops; exp_info.size = gem_obj->size; exp_info.flags = flags; exp_info.priv = gem_obj; + exp_info.resv = obj->resv; if (obj->ops->dmabuf_export) { int ret = obj->ops->dmabuf_export(obj); @@ -272,12 +229,7 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev, return ERR_PTR(ret); } - dma_buf = drm_gem_dmabuf_export(dev, &exp_info); - if (IS_ERR(dma_buf)) - return dma_buf; - - export_fences(obj, dma_buf); - return dma_buf; + return drm_gem_dmabuf_export(dev, &exp_info); } static struct sg_table * @@ -335,6 +287,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, drm_gem_private_object_init(dev, &obj->base, dma_buf->size); i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops); obj->base.import_attach = attach; + obj->resv = dma_buf->resv; /* We use GTT as shorthand for a coherent domain, one that is * neither in the GPU cache nor in the CPU cache, where all diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.h b/drivers/gpu/drm/i915/i915_gem_dmabuf.h deleted file mode 100644 index 91315557e421..000000000000 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef _I915_GEM_DMABUF_H_ -#define _I915_GEM_DMABUF_H_ - -#include - -static inline struct reservation_object * -i915_gem_object_get_dmabuf_resv(struct drm_i915_gem_object *obj) -{ - struct dma_buf *dma_buf; - - if (obj->base.dma_buf) - dma_buf = obj->base.dma_buf; - else if (obj->base.import_attach) - dma_buf = obj->base.import_attach->dmabuf; - else - return NULL; - - return dma_buf->resv; -} - -#endif diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index d95c4e02eeb9..c35e847bb8bc 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -34,7 +34,6 @@ #include #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "i915_trace.h" #include "intel_drv.h" #include "intel_frontbuffer.h" @@ -1101,45 +1100,20 @@ err: return ret; } -static unsigned int eb_other_engines(struct drm_i915_gem_request *req) -{ - unsigned int mask; - - mask = ~intel_engine_flag(req->engine) & I915_BO_ACTIVE_MASK; - mask <<= I915_BO_ACTIVE_SHIFT; - - return mask; -} - static int i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req, struct list_head *vmas) { - const unsigned int other_rings = eb_other_engines(req); struct i915_vma *vma; int ret; list_for_each_entry(vma, vmas, exec_list) { struct drm_i915_gem_object *obj = vma->obj; - struct reservation_object *resv; - if (obj->flags & other_rings) { - ret = i915_gem_request_await_object - (req, obj, obj->base.pending_write_domain); - if (ret) - return ret; - } - - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - ret = i915_sw_fence_await_reservation - (&req->submit, resv, &i915_fence_ops, - obj->base.pending_write_domain, - I915_FENCE_TIMEOUT, - GFP_KERNEL | __GFP_NOWARN); - if (ret < 0) - return ret; - } + ret = i915_gem_request_await_object + (req, obj, obj->base.pending_write_domain); + if (ret) + return ret; if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) i915_gem_clflush_object(obj, false); @@ -1281,8 +1255,6 @@ void i915_vma_move_to_active(struct i915_vma *vma, GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - obj->mm.dirty = true; /* be paranoid */ - /* Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is * vital here: mark_active adds to the start of the callback list, @@ -1290,11 +1262,14 @@ void i915_vma_move_to_active(struct i915_vma *vma, * add the active reference first and queue for it to be dropped * *last*. */ - i915_gem_object_set_active(obj, idx); - i915_gem_active_set(&obj->last_read[idx], req); + if (!i915_vma_is_active(vma)) + obj->active_count++; + i915_vma_set_active(vma, idx); + i915_gem_active_set(&vma->last_read[idx], req); + list_move_tail(&vma->vm_link, &vma->vm->active_list); if (flags & EXEC_OBJECT_WRITE) { - i915_gem_active_set(&obj->last_write, req); + i915_gem_active_set(&vma->last_write, req); intel_fb_obj_invalidate(obj, ORIGIN_CS); @@ -1304,21 +1279,13 @@ void i915_vma_move_to_active(struct i915_vma *vma, if (flags & EXEC_OBJECT_NEEDS_FENCE) i915_gem_active_set(&vma->last_fence, req); - - i915_vma_set_active(vma, idx); - i915_gem_active_set(&vma->last_read[idx], req); - list_move_tail(&vma->vm_link, &vma->vm->active_list); } static void eb_export_fence(struct drm_i915_gem_object *obj, struct drm_i915_gem_request *req, unsigned int flags) { - struct reservation_object *resv; - - resv = i915_gem_object_get_dmabuf_resv(obj); - if (!resv) - return; + struct reservation_object *resv = obj->resv; /* Ignore errors from failing to allocate the new fence, we can't * handle an error right now. Worst case should be missed diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 1008209ca797..1b1a459e2b68 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -31,6 +31,7 @@ #include "i915_vgpu.h" #include "i915_trace.h" #include "intel_drv.h" +#include "intel_frontbuffer.h" #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM) @@ -3343,6 +3344,7 @@ i915_vma_retire(struct i915_gem_active *active, const unsigned int idx = rq->engine->id; struct i915_vma *vma = container_of(active, struct i915_vma, last_read[idx]); + struct drm_i915_gem_object *obj = vma->obj; GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); @@ -3353,6 +3355,34 @@ i915_vma_retire(struct i915_gem_active *active, list_move_tail(&vma->vm_link, &vma->vm->inactive_list); if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma))) WARN_ON(i915_vma_unbind(vma)); + + GEM_BUG_ON(!i915_gem_object_is_active(obj)); + if (--obj->active_count) + return; + + /* Bump our place on the bound list to keep it roughly in LRU order + * so that we don't steal from recently used but inactive objects + * (unless we are forced to ofc!) + */ + if (obj->bind_count) + list_move_tail(&obj->global_list, &rq->i915->mm.bound_list); + + obj->mm.dirty = true; /* be paranoid */ + + if (i915_gem_object_has_active_reference(obj)) { + i915_gem_object_clear_active_reference(obj); + i915_gem_object_put(obj); + } +} + +static void +i915_ggtt_retire__write(struct i915_gem_active *active, + struct drm_i915_gem_request *request) +{ + struct i915_vma *vma = + container_of(active, struct i915_vma, last_write); + + intel_fb_obj_flush(vma->obj, true, ORIGIN_CS); } void i915_vma_destroy(struct i915_vma *vma) @@ -3396,6 +3426,8 @@ __i915_vma_create(struct drm_i915_gem_object *obj, INIT_LIST_HEAD(&vma->exec_list); for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) init_request_active(&vma->last_read[i], i915_vma_retire); + init_request_active(&vma->last_write, + i915_is_ggtt(vm) ? i915_ggtt_retire__write : NULL); init_request_active(&vma->last_fence, NULL); list_add(&vma->vm_link, &vm->unbound_list); vma->vm = vm; diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index dbe6a6cec20d..9f0327e5176a 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -211,6 +211,7 @@ struct i915_vma { unsigned int active; struct i915_gem_active last_read[I915_NUM_ENGINES]; + struct i915_gem_active last_write; struct i915_gem_active last_fence; /** diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index d234c28cbb9f..01a7fa513b4a 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -196,6 +196,8 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) } i915_gem_context_put(request->ctx); + + dma_fence_signal(&request->fence); i915_gem_request_put(request); } @@ -553,33 +555,41 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to, struct drm_i915_gem_object *obj, bool write) { - struct i915_gem_active *active; - unsigned long active_mask; - int idx; + struct dma_fence *excl; + int ret = 0; if (write) { - active_mask = i915_gem_object_get_active(obj); - active = obj->last_read; + struct dma_fence **shared; + unsigned int count, i; + + ret = reservation_object_get_fences_rcu(obj->resv, + &excl, &count, &shared); + if (ret) + return ret; + + for (i = 0; i < count; i++) { + ret = i915_gem_request_await_dma_fence(to, shared[i]); + if (ret) + break; + + dma_fence_put(shared[i]); + } + + for (; i < count; i++) + dma_fence_put(shared[i]); + kfree(shared); } else { - active_mask = 1; - active = &obj->last_write; + excl = reservation_object_get_excl_rcu(obj->resv); } - for_each_active(active_mask, idx) { - struct drm_i915_gem_request *request; - int ret; - - request = i915_gem_active_peek(&active[idx], - &obj->base.dev->struct_mutex); - if (!request) - continue; + if (excl) { + if (ret == 0) + ret = i915_gem_request_await_dma_fence(to, excl); - ret = i915_gem_request_await_request(to, request); - if (ret) - return ret; + dma_fence_put(excl); } - return 0; + return ret; } static void i915_gem_mark_busy(const struct intel_engine_cs *engine) diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h index 602234f91583..a51d596a60ac 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.h +++ b/drivers/gpu/drm/i915/i915_gem_request.h @@ -554,22 +554,7 @@ i915_gem_active_isset(const struct i915_gem_active *active) } /** - * i915_gem_active_is_idle - report whether the active tracker is idle - * @active - the active tracker - * - * i915_gem_active_is_idle() returns true if the active tracker is currently - * unassigned or if the request is complete (but not yet retired). Requires - * the caller to hold struct_mutex (but that can be relaxed if desired). - */ -static inline bool -i915_gem_active_is_idle(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return !i915_gem_active_peek(active, mutex); -} - -/** - * i915_gem_active_wait- waits until the request is completed + * i915_gem_active_wait - waits until the request is completed * @active - the active request on which to wait * @flags - how to wait * @timeout - how long to wait at most @@ -639,24 +624,6 @@ i915_gem_active_retire(struct i915_gem_active *active, return 0; } -/* Convenience functions for peeking at state inside active's request whilst - * guarded by the struct_mutex. - */ - -static inline uint32_t -i915_gem_active_get_seqno(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return i915_gem_request_get_seqno(i915_gem_active_peek(active, mutex)); -} - -static inline struct intel_engine_cs * -i915_gem_active_get_engine(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return i915_gem_request_get_engine(i915_gem_active_peek(active, mutex)); -} - #define for_each_active(mask, idx) \ for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 5bbb37209aa5..aa8dadcc669f 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -887,9 +887,9 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->name = obj->base.name; for (i = 0; i < I915_NUM_ENGINES; i++) - err->rseqno[i] = __active_get_seqno(&obj->last_read[i]); - err->wseqno = __active_get_seqno(&obj->last_write); - err->engine = __active_get_engine_id(&obj->last_write); + err->rseqno[i] = __active_get_seqno(&vma->last_read[i]); + err->wseqno = __active_get_seqno(&vma->last_write); + err->engine = __active_get_engine_id(&vma->last_write); err->gtt_offset = vma->node.start; err->read_domains = obj->base.read_domains; diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c index c762ae549a1c..cb5594411bb6 100644 --- a/drivers/gpu/drm/i915/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/intel_atomic_plane.c @@ -84,7 +84,6 @@ intel_plane_duplicate_state(struct drm_plane *plane) state = &intel_state->base; __drm_atomic_helper_plane_duplicate_state(plane, state); - intel_state->wait_req = NULL; return state; } @@ -101,7 +100,6 @@ void intel_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state) { - WARN_ON(state && to_intel_plane_state(state)->wait_req); drm_atomic_helper_plane_destroy_state(plane, state); } diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 87135d93e828..622f733b9347 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -37,7 +37,6 @@ #include "intel_frontbuffer.h" #include #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "intel_dsi.h" #include "i915_trace.h" #include @@ -11967,8 +11966,6 @@ static int intel_gen7_queue_flip(struct drm_device *dev, static bool use_mmio_flip(struct intel_engine_cs *engine, struct drm_i915_gem_object *obj) { - struct reservation_object *resv; - /* * This is not being used for older platforms, because * non-availability of flip done interrupt forces us to use @@ -11990,12 +11987,7 @@ static bool use_mmio_flip(struct intel_engine_cs *engine, else if (i915.enable_execlists) return true; - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv && !reservation_object_test_signaled_rcu(resv, false)) - return true; - - return engine != i915_gem_active_get_engine(&obj->last_write, - &obj->base.dev->struct_mutex); + return engine != i915_gem_object_last_write_engine(obj); } static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, @@ -12068,17 +12060,8 @@ static void intel_mmio_flip_work_func(struct work_struct *w) struct intel_framebuffer *intel_fb = to_intel_framebuffer(crtc->base.primary->fb); struct drm_i915_gem_object *obj = intel_fb->obj; - struct reservation_object *resv; - if (work->flip_queued_req) - WARN_ON(i915_wait_request(work->flip_queued_req, - 0, MAX_SCHEDULE_TIMEOUT) < 0); - - /* For framebuffer backed by dmabuf, wait for fence */ - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) - WARN_ON(reservation_object_wait_timeout_rcu(resv, false, false, - MAX_SCHEDULE_TIMEOUT) < 0); + WARN_ON(i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT, NULL) < 0); intel_pipe_update_start(crtc); @@ -12279,8 +12262,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, } else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) { engine = dev_priv->engine[BCS]; } else if (INTEL_INFO(dev)->gen >= 7) { - engine = i915_gem_active_get_engine(&obj->last_write, - &obj->base.dev->struct_mutex); + engine = i915_gem_object_last_write_engine(obj); if (engine == NULL || engine->id != RCS) engine = dev_priv->engine[BCS]; } else { @@ -12312,9 +12294,6 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, if (mmio_flip) { INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func); - - work->flip_queued_req = i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); queue_work(system_unbound_wq, &work->mmio_work); } else { request = i915_gem_request_alloc(engine, engine->last_context); @@ -14154,13 +14133,10 @@ static int intel_atomic_check(struct drm_device *dev, } static int intel_atomic_prepare_commit(struct drm_device *dev, - struct drm_atomic_state *state, - bool nonblock) + struct drm_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_plane_state *plane_state; struct drm_crtc_state *crtc_state; - struct drm_plane *plane; struct drm_crtc *crtc; int i, ret; @@ -14183,30 +14159,6 @@ static int intel_atomic_prepare_commit(struct drm_device *dev, ret = drm_atomic_helper_prepare_planes(dev, state); mutex_unlock(&dev->struct_mutex); - if (!ret && !nonblock) { - for_each_plane_in_state(state, plane, plane_state, i) { - struct intel_plane_state *intel_plane_state = - to_intel_plane_state(plane_state); - long timeout; - - if (!intel_plane_state->wait_req) - continue; - - timeout = i915_wait_request(intel_plane_state->wait_req, - I915_WAIT_INTERRUPTIBLE, - MAX_SCHEDULE_TIMEOUT); - if (timeout < 0) { - /* Any hang should be swallowed by the wait */ - WARN_ON(timeout == -EIO); - mutex_lock(&dev->struct_mutex); - drm_atomic_helper_cleanup_planes(dev, state); - mutex_unlock(&dev->struct_mutex); - ret = timeout; - break; - } - } - } - return ret; } @@ -14400,26 +14352,11 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) struct drm_crtc_state *old_crtc_state; struct drm_crtc *crtc; struct intel_crtc_state *intel_cstate; - struct drm_plane *plane; - struct drm_plane_state *plane_state; bool hw_check = intel_state->modeset; unsigned long put_domains[I915_MAX_PIPES] = {}; unsigned crtc_vblank_mask = 0; int i; - for_each_plane_in_state(state, plane, plane_state, i) { - struct intel_plane_state *intel_plane_state = - to_intel_plane_state(plane->state); - - if (!intel_plane_state->wait_req) - continue; - - /* EIO should be eaten, and we can't get interrupted in the - * worker, and blocking commits have waited already. */ - WARN_ON(i915_wait_request(intel_plane_state->wait_req, - 0, MAX_SCHEDULE_TIMEOUT) < 0); - } - drm_atomic_helper_wait_for_dependencies(state); if (intel_state->modeset) { @@ -14626,7 +14563,7 @@ static int intel_atomic_commit(struct drm_device *dev, INIT_WORK(&state->commit_work, intel_atomic_commit_work); - ret = intel_atomic_prepare_commit(dev, state, nonblock); + ret = intel_atomic_prepare_commit(dev, state); if (ret) { DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret); return ret; @@ -14759,7 +14696,7 @@ intel_prepare_plane_fb(struct drm_plane *plane, struct drm_framebuffer *fb = new_state->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb); - struct reservation_object *resv; + long lret; int ret = 0; if (!obj && !old_obj) @@ -14797,39 +14734,34 @@ intel_prepare_plane_fb(struct drm_plane *plane, return 0; /* For framebuffer backed by dmabuf, wait for fence */ - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - long lret; + lret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT, + NULL); + if (lret == -ERESTARTSYS) + return lret; - lret = reservation_object_wait_timeout_rcu(resv, false, true, - MAX_SCHEDULE_TIMEOUT); - if (lret == -ERESTARTSYS) - return lret; - - WARN(lret < 0, "waiting returns %li\n", lret); - } + WARN(lret < 0, "waiting returns %li\n", lret); if (plane->type == DRM_PLANE_TYPE_CURSOR && INTEL_INFO(dev)->cursor_needs_physical) { int align = IS_I830(dev_priv) ? 16 * 1024 : 256; ret = i915_gem_object_attach_phys(obj, align); - if (ret) + if (ret) { DRM_DEBUG_KMS("failed to attach phys object\n"); + return ret; + } } else { struct i915_vma *vma; vma = intel_pin_and_fence_fb_obj(fb, new_state->rotation); - if (IS_ERR(vma)) - ret = PTR_ERR(vma); - } - - if (ret == 0) { - to_intel_plane_state(new_state)->wait_req = - i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); + if (IS_ERR(vma)) { + DRM_DEBUG_KMS("failed to pin object\n"); + return PTR_ERR(vma); + } } - return ret; + return 0; } /** @@ -14847,7 +14779,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane, { struct drm_device *dev = plane->dev; struct intel_plane_state *old_intel_state; - struct intel_plane_state *intel_state = to_intel_plane_state(plane->state); struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb); struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb); @@ -14859,9 +14790,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane, if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR || !INTEL_INFO(dev)->cursor_needs_physical)) intel_unpin_fb_obj(old_state->fb, old_state->rotation); - - i915_gem_request_assign(&intel_state->wait_req, NULL); - i915_gem_request_assign(&old_intel_state->wait_req, NULL); } int diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index ace222c74f71..ec7fa592b20a 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -401,9 +401,6 @@ struct intel_plane_state { int scaler_id; struct drm_intel_sprite_colorkey ckey; - - /* async flip related structures */ - struct drm_i915_gem_request *wait_req; }; struct intel_initial_plane_config { -- cgit v1.2.3 From 73cb97010d4fdd2a29f00cac14d206c7641c23d2 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:46 +0100 Subject: drm/i915: Combine seqno + tracking into a global timeline struct Our timelines are more than just a seqno. They also provide an ordered list of requests to be executed. Due to the restriction of handling individual address spaces, we are limited to a timeline per address space but we use a fence context per engine within. Our first step to introducing independent timelines per context (i.e. to allow each context to have a queue of requests to execute that have a defined set of dependencies on other requests) is to provide a timeline abstraction for the global execution queue. Signed-off-by: Chris Wilson Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-23-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_debugfs.c | 33 +++++------- drivers/gpu/drm/i915/i915_drv.c | 6 ++- drivers/gpu/drm/i915/i915_drv.h | 9 ++-- drivers/gpu/drm/i915/i915_gem.c | 72 ++++++++++++++++++++------ drivers/gpu/drm/i915/i915_gem.h | 2 + drivers/gpu/drm/i915/i915_gem_request.c | 81 ++++++++++++++++++------------ drivers/gpu/drm/i915/i915_gem_request.h | 1 + drivers/gpu/drm/i915/i915_gem_timeline.c | 64 +++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_timeline.h | 70 ++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gpu_error.c | 6 +-- drivers/gpu/drm/i915/i915_guc_submission.c | 3 +- drivers/gpu/drm/i915/i915_irq.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 15 +++--- drivers/gpu/drm/i915/intel_ringbuffer.h | 36 ++----------- 15 files changed, 286 insertions(+), 115 deletions(-) create mode 100644 drivers/gpu/drm/i915/i915_gem_timeline.c create mode 100644 drivers/gpu/drm/i915/i915_gem_timeline.h (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 7faa04c91e1a..240ce9a8d68e 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -42,6 +42,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_shrinker.o \ i915_gem_stolen.o \ i915_gem_tiling.o \ + i915_gem_timeline.o \ i915_gem_userptr.o \ i915_trace_points.o \ intel_breadcrumbs.o \ diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index b6325065c8e8..3a0ea5eace37 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -552,7 +552,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) seq_printf(m, "Flip queued on %s at seqno %x, next seqno %x [current breadcrumb %x], completed? %d\n", engine->name, i915_gem_request_get_seqno(work->flip_queued_req), - dev_priv->next_seqno, + dev_priv->gt.global_timeline.next_seqno, intel_engine_get_seqno(engine), i915_gem_request_completed(work->flip_queued_req)); } else @@ -662,13 +662,13 @@ static int i915_gem_request_info(struct seq_file *m, void *data) int count; count = 0; - list_for_each_entry(req, &engine->request_list, link) + list_for_each_entry(req, &engine->timeline->requests, link) count++; if (count == 0) continue; seq_printf(m, "%s requests: %d\n", engine->name, count); - list_for_each_entry(req, &engine->request_list, link) + list_for_each_entry(req, &engine->timeline->requests, link) print_request(m, req, " "); any++; @@ -1052,15 +1052,8 @@ static int i915_next_seqno_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - int ret; - - ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex); - if (ret) - return ret; - - *val = dev_priv->next_seqno; - mutex_unlock(&dev_priv->drm.struct_mutex); + *val = READ_ONCE(dev_priv->gt.global_timeline.next_seqno); return 0; } @@ -1075,7 +1068,7 @@ i915_next_seqno_set(void *data, u64 val) if (ret) return ret; - ret = i915_gem_set_seqno(dev, val); + ret = i915_gem_set_global_seqno(dev, val); mutex_unlock(&dev->struct_mutex); return ret; @@ -1364,7 +1357,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) seq_printf(m, "\tseqno = %x [current %x, last %x]\n", engine->hangcheck.seqno, seqno[id], - engine->last_submitted_seqno); + engine->timeline->last_submitted_seqno); seq_printf(m, "\twaiters? %s, fake irq active? %s\n", yesno(intel_engine_has_waiter(engine)), yesno(test_bit(engine->id, @@ -3181,7 +3174,7 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_printf(m, "%s\n", engine->name); seq_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [score %d]\n", intel_engine_get_seqno(engine), - engine->last_submitted_seqno, + engine->timeline->last_submitted_seqno, engine->hangcheck.seqno, engine->hangcheck.score); @@ -3189,14 +3182,14 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_printf(m, "\tRequests:\n"); - rq = list_first_entry(&engine->request_list, - struct drm_i915_gem_request, link); - if (&rq->link != &engine->request_list) + rq = list_first_entry(&engine->timeline->requests, + struct drm_i915_gem_request, link); + if (&rq->link != &engine->timeline->requests) print_request(m, rq, "\t\tfirst "); - rq = list_last_entry(&engine->request_list, - struct drm_i915_gem_request, link); - if (&rq->link != &engine->request_list) + rq = list_last_entry(&engine->timeline->requests, + struct drm_i915_gem_request, link); + if (&rq->link != &engine->timeline->requests) print_request(m, rq, "\t\tlast "); rq = i915_gem_find_active_request(engine); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 91cd7b296c0f..839ce2ae38fa 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -831,7 +831,9 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv, intel_init_display_hooks(dev_priv); intel_init_clock_gating_hooks(dev_priv); intel_init_audio_hooks(dev_priv); - i915_gem_load_init(&dev_priv->drm); + ret = i915_gem_load_init(&dev_priv->drm); + if (ret < 0) + goto err_gvt; intel_display_crc_init(dev_priv); @@ -841,6 +843,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv, return 0; +err_gvt: + intel_gvt_cleanup(dev_priv); err_workqueues: i915_workqueues_cleanup(dev_priv); return ret; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 693ee69a4715..f0f68f64d09c 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -63,6 +63,7 @@ #include "i915_gem_gtt.h" #include "i915_gem_render_state.h" #include "i915_gem_request.h" +#include "i915_gem_timeline.h" #include "intel_gvt.h" @@ -1830,7 +1831,6 @@ struct drm_i915_private { struct i915_gem_context *kernel_context; struct intel_engine_cs *engine[I915_NUM_ENGINES]; struct i915_vma *semaphore; - u32 next_seqno; struct drm_dma_handle *status_page_dmah; struct resource mch_res; @@ -2090,6 +2090,9 @@ struct drm_i915_private { void (*resume)(struct drm_i915_private *); void (*cleanup_engine)(struct intel_engine_cs *engine); + struct list_head timelines; + struct i915_gem_timeline global_timeline; + /** * Is the GPU currently considered idle, or busy executing * userspace requests? Whilst idle, we allow runtime power @@ -3175,7 +3178,7 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); -void i915_gem_load_init(struct drm_device *dev); +int i915_gem_load_init(struct drm_device *dev); void i915_gem_load_cleanup(struct drm_device *dev); void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); int i915_gem_freeze(struct drm_i915_private *dev_priv); @@ -3347,7 +3350,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, struct drm_i915_gem_object *new, unsigned frontbuffer_bits); -int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); +int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); struct drm_i915_gem_request * i915_gem_find_active_request(struct intel_engine_cs *engine); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 1161a21ec810..525360219bbb 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -371,7 +371,7 @@ out: if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq)) i915_gem_request_retire_upto(rq); - if (rps && rq->fence.seqno == rq->engine->last_submitted_seqno) { + if (rps && rq->fence.seqno == rq->timeline->last_submitted_seqno) { /* The GPU is now idle and this client has stalled. * Since no other client has submitted a request in the * meantime, assume that this client is the only one @@ -2563,7 +2563,7 @@ i915_gem_find_active_request(struct intel_engine_cs *engine) * extra delay for a recent interrupt is pointless. Hence, we do * not need an engine->irq_seqno_barrier() before the seqno reads. */ - list_for_each_entry(request, &engine->request_list, link) { + list_for_each_entry(request, &engine->timeline->requests, link) { if (i915_gem_request_completed(request)) continue; @@ -2632,7 +2632,7 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) if (i915_gem_context_is_default(incomplete_ctx)) return; - list_for_each_entry_continue(request, &engine->request_list, link) + list_for_each_entry_continue(request, &engine->timeline->requests, link) if (request->ctx == incomplete_ctx) reset_request(request); } @@ -2671,7 +2671,8 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) * (lockless) lookup doesn't try and wait upon the request as we * reset it. */ - intel_engine_init_seqno(engine, engine->last_submitted_seqno); + intel_engine_init_global_seqno(engine, + engine->timeline->last_submitted_seqno); /* * Clear the execlists queue up before freeing the requests, as those @@ -2979,18 +2980,26 @@ destroy: return 0; } -int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, - unsigned int flags) +static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags) { - struct intel_engine_cs *engine; - enum intel_engine_id id; - int ret; + int ret, i; - for_each_engine(engine, dev_priv, id) { - if (engine->last_context == NULL) - continue; + for (i = 0; i < ARRAY_SIZE(tl->engine); i++) { + ret = i915_gem_active_wait(&tl->engine[i].last_request, flags); + if (ret) + return ret; + } - ret = intel_engine_idle(engine, flags); + return 0; +} + +int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags) +{ + struct i915_gem_timeline *tl; + int ret; + + list_for_each_entry(tl, &i915->gt.timelines, link) { + ret = wait_for_timeline(tl, flags); if (ret) return ret; } @@ -4680,21 +4689,32 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv) i915_gem_detect_bit_6_swizzle(dev); } -void +int i915_gem_load_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + int err; dev_priv->objects = kmem_cache_create("i915_gem_object", sizeof(struct drm_i915_gem_object), 0, SLAB_HWCACHE_ALIGN, NULL); + if (!dev_priv->objects) { + err = -ENOMEM; + goto err_out; + } + dev_priv->vmas = kmem_cache_create("i915_gem_vma", sizeof(struct i915_vma), 0, SLAB_HWCACHE_ALIGN, NULL); + if (!dev_priv->vmas) { + err = -ENOMEM; + goto err_objects; + } + dev_priv->requests = kmem_cache_create("i915_gem_request", sizeof(struct drm_i915_gem_request), 0, @@ -4702,6 +4722,19 @@ i915_gem_load_init(struct drm_device *dev) SLAB_RECLAIM_ACCOUNT | SLAB_DESTROY_BY_RCU, NULL); + if (!dev_priv->requests) { + err = -ENOMEM; + goto err_vmas; + } + + mutex_lock(&dev_priv->drm.struct_mutex); + INIT_LIST_HEAD(&dev_priv->gt.timelines); + err = i915_gem_timeline_init(dev_priv, + &dev_priv->gt.global_timeline, + "[execution]"); + mutex_unlock(&dev_priv->drm.struct_mutex); + if (err) + goto err_requests; INIT_LIST_HEAD(&dev_priv->context_list); INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work); @@ -4726,6 +4759,17 @@ i915_gem_load_init(struct drm_device *dev) atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0); spin_lock_init(&dev_priv->fb_tracking.lock); + + return 0; + +err_requests: + kmem_cache_destroy(dev_priv->requests); +err_vmas: + kmem_cache_destroy(dev_priv->vmas); +err_objects: + kmem_cache_destroy(dev_priv->objects); +err_out: + return err; } void i915_gem_load_cleanup(struct drm_device *dev) diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index 8292e797d9b5..735580d72eb1 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -31,4 +31,6 @@ #define GEM_BUG_ON(expr) #endif +#define I915_NUM_ENGINES 5 + #endif /* __I915_GEM_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index 01a7fa513b4a..16d38f87f0a7 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -40,7 +40,7 @@ static const char *i915_fence_get_timeline_name(struct dma_fence *fence) * multiple execution contexts (fence contexts) as we allow * engines within a single timeline to execute in parallel. */ - return "global"; + return to_request(fence)->timeline->common->name; } static bool i915_fence_signaled(struct dma_fence *fence) @@ -211,7 +211,7 @@ void i915_gem_request_retire_upto(struct drm_i915_gem_request *req) return; do { - tmp = list_first_entry(&engine->request_list, + tmp = list_first_entry(&engine->timeline->requests, typeof(*tmp), link); i915_gem_request_retire(tmp); @@ -238,37 +238,39 @@ static int i915_gem_check_wedge(struct drm_i915_private *dev_priv) return 0; } -static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno) +static int i915_gem_init_global_seqno(struct drm_i915_private *dev_priv, + u32 seqno) { + struct i915_gem_timeline *timeline = &dev_priv->gt.global_timeline; struct intel_engine_cs *engine; enum intel_engine_id id; int ret; /* Carefully retire all requests without writing to the rings */ - for_each_engine(engine, dev_priv, id) { - ret = intel_engine_idle(engine, - I915_WAIT_INTERRUPTIBLE | - I915_WAIT_LOCKED); - if (ret) - return ret; - } + ret = i915_gem_wait_for_idle(dev_priv, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED); + if (ret) + return ret; + i915_gem_retire_requests(dev_priv); /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ - if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) { + if (!i915_seqno_passed(seqno, timeline->next_seqno)) { while (intel_kick_waiters(dev_priv) || intel_kick_signalers(dev_priv)) yield(); + yield(); } /* Finally reset hw state */ for_each_engine(engine, dev_priv, id) - intel_engine_init_seqno(engine, seqno); + intel_engine_init_global_seqno(engine, seqno); return 0; } -int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) +int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno) { struct drm_i915_private *dev_priv = to_i915(dev); int ret; @@ -281,28 +283,31 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) /* HWS page needs to be set less than what we * will inject to ring */ - ret = i915_gem_init_seqno(dev_priv, seqno - 1); + ret = i915_gem_init_global_seqno(dev_priv, seqno - 1); if (ret) return ret; - dev_priv->next_seqno = seqno; + dev_priv->gt.global_timeline.next_seqno = seqno; return 0; } -static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno) +static int i915_gem_get_global_seqno(struct drm_i915_private *dev_priv, + u32 *seqno) { + struct i915_gem_timeline *tl = &dev_priv->gt.global_timeline; + /* reserve 0 for non-seqno */ - if (unlikely(dev_priv->next_seqno == 0)) { + if (unlikely(tl->next_seqno == 0)) { int ret; - ret = i915_gem_init_seqno(dev_priv, 0); + ret = i915_gem_init_global_seqno(dev_priv, 0); if (ret) return ret; - dev_priv->next_seqno = 1; + tl->next_seqno = 1; } - *seqno = dev_priv->next_seqno++; + *seqno = tl->next_seqno++; return 0; } @@ -311,13 +316,14 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) { struct drm_i915_gem_request *request = container_of(fence, typeof(*request), submit); + struct intel_engine_cs *engine = request->engine; /* Will be called from irq-context when using foreign DMA fences */ switch (state) { case FENCE_COMPLETE: - request->engine->last_submitted_seqno = request->fence.seqno; - request->engine->submit_request(request); + engine->timeline->last_submitted_seqno = request->fence.seqno; + engine->submit_request(request); break; case FENCE_FREE: @@ -357,7 +363,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, return ERR_PTR(ret); /* Move the oldest request to the slab-cache (if not in use!) */ - req = list_first_entry_or_null(&engine->request_list, + req = list_first_entry_or_null(&engine->timeline->requests, typeof(*req), link); if (req && i915_gem_request_completed(req)) i915_gem_request_retire(req); @@ -394,15 +400,17 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, if (!req) return ERR_PTR(-ENOMEM); - ret = i915_gem_get_seqno(dev_priv, &seqno); + ret = i915_gem_get_global_seqno(dev_priv, &seqno); if (ret) goto err; + req->timeline = engine->timeline; + spin_lock_init(&req->lock); dma_fence_init(&req->fence, &i915_fence_ops, &req->lock, - engine->fence_context, + req->timeline->fence_context, seqno); i915_sw_fence_init(&req->submit, submit_notify); @@ -457,9 +465,16 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, GEM_BUG_ON(to == from); - if (to->engine == from->engine) + if (to->timeline == from->timeline) return 0; + if (to->engine == from->engine) { + ret = i915_sw_fence_await_sw_fence_gfp(&to->submit, + &from->submit, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + idx = intel_engine_sync_index(from->engine, to->engine); if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx]) return 0; @@ -622,6 +637,7 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) { struct intel_engine_cs *engine = request->engine; struct intel_ring *ring = request->ring; + struct intel_timeline *timeline = request->timeline; struct drm_i915_gem_request *prev; u32 request_start; u32 reserved_tail; @@ -679,17 +695,17 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * see a more recent value in the hws than we are tracking. */ - prev = i915_gem_active_raw(&engine->last_request, + prev = i915_gem_active_raw(&timeline->last_request, &request->i915->drm.struct_mutex); if (prev) i915_sw_fence_await_sw_fence(&request->submit, &prev->submit, &request->submitq); request->emitted_jiffies = jiffies; - request->previous_seqno = engine->last_pending_seqno; - engine->last_pending_seqno = request->fence.seqno; - i915_gem_active_set(&engine->last_request, request); - list_add_tail(&request->link, &engine->request_list); + request->previous_seqno = timeline->last_pending_seqno; + timeline->last_pending_seqno = request->fence.seqno; + i915_gem_active_set(&timeline->last_request, request); + list_add_tail(&request->link, &timeline->requests); list_add_tail(&request->ring_link, &ring->request_list); i915_gem_mark_busy(engine); @@ -899,7 +915,8 @@ static bool engine_retire_requests(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request, *next; - list_for_each_entry_safe(request, next, &engine->request_list, link) { + list_for_each_entry_safe(request, next, + &engine->timeline->requests, link) { if (!i915_gem_request_completed(request)) return false; diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h index a51d596a60ac..4ac30ae93e49 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.h +++ b/drivers/gpu/drm/i915/i915_gem_request.h @@ -81,6 +81,7 @@ struct drm_i915_gem_request { struct i915_gem_context *ctx; struct intel_engine_cs *engine; struct intel_ring *ring; + struct intel_timeline *timeline; struct intel_signal_node signaling; struct i915_sw_fence submit; diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c new file mode 100644 index 000000000000..a1bd03d10852 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_timeline.c @@ -0,0 +1,64 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "i915_drv.h" + +int i915_gem_timeline_init(struct drm_i915_private *i915, + struct i915_gem_timeline *timeline, + const char *name) +{ + unsigned int i; + u64 fences; + + lockdep_assert_held(&i915->drm.struct_mutex); + + timeline->i915 = i915; + timeline->name = kstrdup(name ?: "[kernel]", GFP_KERNEL); + if (!timeline->name) + return -ENOMEM; + + list_add(&timeline->link, &i915->gt.timelines); + + /* Called during early_init before we know how many engines there are */ + fences = dma_fence_context_alloc(ARRAY_SIZE(timeline->engine)); + for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) { + struct intel_timeline *tl = &timeline->engine[i]; + + tl->fence_context = fences++; + tl->common = timeline; + + init_request_active(&tl->last_request, NULL); + INIT_LIST_HEAD(&tl->requests); + } + + return 0; +} + +void i915_gem_timeline_fini(struct i915_gem_timeline *tl) +{ + lockdep_assert_held(&tl->i915->drm.struct_mutex); + + list_del(&tl->link); + kfree(tl->name); +} diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h new file mode 100644 index 000000000000..bfdf0331cc50 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_timeline.h @@ -0,0 +1,70 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#ifndef I915_GEM_TIMELINE_H +#define I915_GEM_TIMELINE_H + +#include + +#include "i915_gem_request.h" + +struct i915_gem_timeline; + +struct intel_timeline { + u64 fence_context; + u32 last_submitted_seqno; + u32 last_pending_seqno; + + /** + * List of breadcrumbs associated with GPU requests currently + * outstanding. + */ + struct list_head requests; + + /* Contains an RCU guarded pointer to the last request. No reference is + * held to the request, users must carefully acquire a reference to + * the request using i915_gem_active_get_request_rcu(), or hold the + * struct_mutex. + */ + struct i915_gem_active last_request; + + struct i915_gem_timeline *common; +}; + +struct i915_gem_timeline { + struct list_head link; + u32 next_seqno; + + struct drm_i915_private *i915; + const char *name; + + struct intel_timeline engine[I915_NUM_ENGINES]; +}; + +int i915_gem_timeline_init(struct drm_i915_private *i915, + struct i915_gem_timeline *tl, + const char *name); +void i915_gem_timeline_fini(struct i915_gem_timeline *tl); + +#endif diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index aa8dadcc669f..12fea57d41fb 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1110,7 +1110,7 @@ static void error_record_engine_registers(struct drm_i915_error_state *error, ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base)); ee->acthd = intel_engine_get_active_head(engine); ee->seqno = intel_engine_get_seqno(engine); - ee->last_seqno = engine->last_submitted_seqno; + ee->last_seqno = engine->timeline->last_submitted_seqno; ee->start = I915_READ_START(engine); ee->head = I915_READ_HEAD(engine); ee->tail = I915_READ_TAIL(engine); @@ -1195,7 +1195,7 @@ static void engine_record_requests(struct intel_engine_cs *engine, count = 0; request = first; - list_for_each_entry_from(request, &engine->request_list, link) + list_for_each_entry_from(request, &engine->timeline->requests, link) count++; if (!count) return; @@ -1208,7 +1208,7 @@ static void engine_record_requests(struct intel_engine_cs *engine, count = 0; request = first; - list_for_each_entry_from(request, &engine->request_list, link) { + list_for_each_entry_from(request, &engine->timeline->requests, link) { if (count >= ee->num_requests) { /* * If the ring request list was changed in diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index 74235ea3950f..cca250e90845 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -1522,7 +1522,8 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv) engine->submit_request = i915_guc_submit; /* Replay the current set of previously submitted requests */ - list_for_each_entry(request, &engine->request_list, link) { + list_for_each_entry(request, + &engine->timeline->requests, link) { client->wq_rsvd += sizeof(struct guc_wq_item); if (i915_sw_fence_done(&request->submit)) i915_guc_submit(request); diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 88239e1b29e4..90d0905592f2 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -3168,7 +3168,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work) acthd = intel_engine_get_active_head(engine); seqno = intel_engine_get_seqno(engine); - submit = READ_ONCE(engine->last_submitted_seqno); + submit = READ_ONCE(engine->timeline->last_submitted_seqno); if (engine->hangcheck.seqno == seqno) { if (i915_seqno_passed(seqno, submit)) { diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c index fd551824adf9..6a3105512d18 100644 --- a/drivers/gpu/drm/i915/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/intel_engine_cs.c @@ -174,7 +174,7 @@ cleanup: return ret; } -void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) +void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno) { struct drm_i915_private *dev_priv = engine->i915; @@ -210,7 +210,9 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); if (engine->irq_seqno_barrier) engine->irq_seqno_barrier(engine); - engine->last_submitted_seqno = seqno; + + GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request)); + engine->timeline->last_submitted_seqno = seqno; engine->hangcheck.seqno = seqno; @@ -225,10 +227,9 @@ void intel_engine_init_hangcheck(struct intel_engine_cs *engine) memset(&engine->hangcheck, 0, sizeof(engine->hangcheck)); } -static void intel_engine_init_requests(struct intel_engine_cs *engine) +static void intel_engine_init_timeline(struct intel_engine_cs *engine) { - init_request_active(&engine->last_request, NULL); - INIT_LIST_HEAD(&engine->request_list); + engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id]; } /** @@ -245,9 +246,7 @@ void intel_engine_setup_common(struct intel_engine_cs *engine) INIT_LIST_HEAD(&engine->execlist_queue); spin_lock_init(&engine->execlist_lock); - engine->fence_context = dma_fence_context_alloc(1); - - intel_engine_init_requests(engine); + intel_engine_init_timeline(engine); intel_engine_init_hangcheck(engine); i915_gem_batch_pool_init(engine, &engine->batch_pool); diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index cb6e96c6cd47..a62e396c8863 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -4,6 +4,7 @@ #include #include "i915_gem_batch_pool.h" #include "i915_gem_request.h" +#include "i915_gem_timeline.h" #define I915_CMD_HASH_ORDER 9 @@ -169,7 +170,6 @@ struct intel_engine_cs { VCS2, /* Keep instances of the same type engine together. */ VECS } id; -#define I915_NUM_ENGINES 5 #define _VCS(n) (VCS + (n)) unsigned int exec_id; enum intel_engine_hw_id { @@ -180,10 +180,10 @@ struct intel_engine_cs { VCS2_HW } hw_id; enum intel_engine_hw_id guc_id; /* XXX same as hw_id? */ - u64 fence_context; u32 mmio_base; unsigned int irq_shift; struct intel_ring *buffer; + struct intel_timeline *timeline; struct intel_render_state *render_state; @@ -346,27 +346,6 @@ struct intel_engine_cs { bool preempt_wa; u32 ctx_desc_template; - /** - * List of breadcrumbs associated with GPU requests currently - * outstanding. - */ - struct list_head request_list; - - /** - * Seqno of request most recently submitted to request_list. - * Used exclusively by hang checker to avoid grabbing lock while - * inspecting request list. - */ - u32 last_submitted_seqno; - u32 last_pending_seqno; - - /* An RCU guarded pointer to the last request. No reference is - * held to the request, users must carefully acquire a reference to - * the request using i915_gem_active_get_rcu(), or hold the - * struct_mutex. - */ - struct i915_gem_active last_request; - struct i915_gem_context *last_context; struct intel_engine_hangcheck hangcheck; @@ -516,20 +495,13 @@ static inline u32 intel_ring_offset(struct intel_ring *ring, u32 value) int __intel_ring_space(int head, int tail, int size); void intel_ring_update_space(struct intel_ring *ring); -void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno); +void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno); void intel_engine_setup_common(struct intel_engine_cs *engine); int intel_engine_init_common(struct intel_engine_cs *engine); int intel_engine_create_scratch(struct intel_engine_cs *engine, int size); void intel_engine_cleanup_common(struct intel_engine_cs *engine); -static inline int intel_engine_idle(struct intel_engine_cs *engine, - unsigned int flags) -{ - /* Wait upon the last request to be completed */ - return i915_gem_active_wait(&engine->last_request, flags); -} - int intel_init_render_ring_buffer(struct intel_engine_cs *engine); int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine); int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine); @@ -619,7 +591,7 @@ unsigned int intel_kick_signalers(struct drm_i915_private *i915); static inline bool intel_engine_is_active(struct intel_engine_cs *engine) { - return i915_gem_active_isset(&engine->last_request); + return i915_gem_active_isset(&engine->timeline->last_request); } #endif /* _INTEL_RINGBUFFER_H_ */ -- cgit v1.2.3 From 65e4760e3920c21073a9d737929dc36df561380f Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:49 +0100 Subject: drm/i915: Introduce a global_seqno for each request Though we will have multiple timelines, we still have a single timeline of execution. This we can use to provide an execution and retirement order of requests. This keeps tracking execution of requests simple, and vital for preserving a single waiter (i.e. so that we can order the waiters so that only the earliest to wakeup need be woken). To accomplish this we distinguish the seqno used to order requests per-context (external) and that used internally for execution. Signed-off-by: Chris Wilson Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-26-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_drv.h | 4 ++-- drivers/gpu/drm/i915/i915_gem.c | 2 +- drivers/gpu/drm/i915/i915_gem_request.c | 19 +++++++++++++----- drivers/gpu/drm/i915/i915_gem_request.h | 32 +++++++++++++++++++++++++----- drivers/gpu/drm/i915/i915_gpu_error.c | 2 +- drivers/gpu/drm/i915/i915_guc_submission.c | 4 ++-- drivers/gpu/drm/i915/i915_trace.h | 8 ++++---- drivers/gpu/drm/i915/intel_breadcrumbs.c | 8 +++++--- drivers/gpu/drm/i915/intel_lrc.c | 4 ++-- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++++++------- 11 files changed, 66 insertions(+), 33 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 3a0ea5eace37..90bc4a89e0d5 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -637,7 +637,7 @@ static void print_request(struct seq_file *m, rcu_read_lock(); task = pid ? pid_task(pid, PIDTYPE_PID) : NULL; seq_printf(m, "%s%x [%x:%x] @ %d: %s [%d]\n", prefix, - rq->fence.seqno, rq->ctx->hw_id, rq->fence.seqno, + rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno, jiffies_to_msecs(jiffies - rq->emitted_jiffies), task ? task->comm : "", task ? task->pid : -1); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index f0f68f64d09c..217674bb1495 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -4050,7 +4050,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) /* Before we do the heavier coherent read of the seqno, * check the value (hopefully) in the CPU cacheline. */ - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; /* Ensure our read of the seqno is coherent so that we @@ -4101,7 +4101,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) wake_up_process(tsk); rcu_read_unlock(); - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; } diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 5ea46a7d991f..f4cfb88bd804 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -2615,7 +2615,7 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) return; DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n", - engine->name, request->fence.seqno); + engine->name, request->global_seqno); /* Setup the CS to resume from the breadcrumb of the hung request */ engine->reset_hw(engine, request); diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index 03ae85a1eefb..311cf3fac2e0 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -376,7 +376,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * of being read by __i915_gem_active_get_rcu(). As such, * we have to be very careful when overwriting the contents. During * the RCU lookup, we change chase the request->engine pointer, - * read the request->fence.seqno and increment the reference count. + * read the request->global_seqno and increment the reference count. * * The reference count is incremented atomically. If it is zero, * the lookup knows the request is unallocated and complete. Otherwise, @@ -418,6 +418,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, INIT_LIST_HEAD(&req->active_list); req->i915 = dev_priv; req->engine = engine; + req->global_seqno = seqno; req->ctx = i915_gem_context_get(ctx); /* No zalloc, must clear what we need by hand */ @@ -475,8 +476,15 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret < 0 ? ret : 0; } + if (!from->global_seqno) { + ret = i915_sw_fence_await_dma_fence(&to->submit, + &from->fence, 0, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + idx = intel_engine_sync_index(from->engine, to->engine); - if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx]) + if (from->global_seqno <= from->engine->semaphore.sync_seqno[idx]) return 0; trace_i915_gem_ring_sync_to(to, from); @@ -494,7 +502,7 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret; } - from->engine->semaphore.sync_seqno[idx] = from->fence.seqno; + from->engine->semaphore.sync_seqno[idx] = from->global_seqno; return 0; } @@ -774,7 +782,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req, timeout_us += local_clock_us(&cpu); do { - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; if (signal_pending_state(state, current)) @@ -883,6 +891,7 @@ long i915_wait_request(struct drm_i915_gem_request *req, GEM_BUG_ON(!i915_sw_fence_done(&req->submit)); } + GEM_BUG_ON(!req->global_seqno); /* Optimistic short spin before touching IRQs */ if (i915_spin_request(req, state, 5)) @@ -892,7 +901,7 @@ long i915_wait_request(struct drm_i915_gem_request *req, if (flags & I915_WAIT_LOCKED) add_wait_queue(&req->i915->gpu_error.wait_queue, &reset); - intel_wait_init(&wait, req->fence.seqno); + intel_wait_init(&wait, req->global_seqno); if (intel_engine_add_wait(req->engine, &wait)) /* In order to check that we haven't missed the interrupt * as we enabled it, we need to kick ourselves to do a diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h index 4ac30ae93e49..75f8360b3421 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.h +++ b/drivers/gpu/drm/i915/i915_gem_request.h @@ -87,6 +87,8 @@ struct drm_i915_gem_request { struct i915_sw_fence submit; wait_queue_t submitq; + u32 global_seqno; + /** GEM sequence number associated with the previous request, * when the HWS breadcrumb is equal to this the GPU is processing * this request. @@ -163,7 +165,7 @@ void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); static inline u32 i915_gem_request_get_seqno(struct drm_i915_gem_request *req) { - return req ? req->fence.seqno : 0; + return req ? req->global_seqno : 0; } static inline struct intel_engine_cs * @@ -248,17 +250,37 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2) } static inline bool -i915_gem_request_started(const struct drm_i915_gem_request *req) +__i915_gem_request_started(const struct drm_i915_gem_request *req) { + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), req->previous_seqno); } static inline bool -i915_gem_request_completed(const struct drm_i915_gem_request *req) +i915_gem_request_started(const struct drm_i915_gem_request *req) { + if (!req->global_seqno) + return false; + + return __i915_gem_request_started(req); +} + +static inline bool +__i915_gem_request_completed(const struct drm_i915_gem_request *req) +{ + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), - req->fence.seqno); + req->global_seqno); +} + +static inline bool +i915_gem_request_completed(const struct drm_i915_gem_request *req) +{ + if (!req->global_seqno) + return false; + + return __i915_gem_request_completed(req); } bool __i915_spin_request(const struct drm_i915_gem_request *request, @@ -266,7 +288,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *request, static inline bool i915_spin_request(const struct drm_i915_gem_request *request, int state, unsigned long timeout_us) { - return (i915_gem_request_started(request) && + return (__i915_gem_request_started(request) && __i915_spin_request(request, state, timeout_us)); } diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 12fea57d41fb..9aa197ca6210 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1176,7 +1176,7 @@ static void record_request(struct drm_i915_gem_request *request, struct drm_i915_error_request *erq) { erq->context = request->ctx->hw_id; - erq->seqno = request->fence.seqno; + erq->seqno = request->global_seqno; erq->jiffies = request->emitted_jiffies; erq->head = request->head; erq->tail = request->tail; diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index cca250e90845..857ef914cae7 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -554,7 +554,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc, wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine); wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT; - wqi->fence_id = rq->fence.seqno; + wqi->fence_id = rq->global_seqno; kunmap_atomic(base); } @@ -655,7 +655,7 @@ static void i915_guc_submit(struct drm_i915_gem_request *rq) client->b_fail += 1; guc->submissions[engine_id] += 1; - guc->last_seqno[engine_id] = rq->fence.seqno; + guc->last_seqno[engine_id] = rq->global_seqno; spin_unlock(&client->wq_lock); } diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 5c912c25f7d3..c5d210ebaa9a 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -466,7 +466,7 @@ TRACE_EVENT(i915_gem_ring_sync_to, __entry->dev = from->i915->drm.primary->index; __entry->sync_from = from->engine->id; __entry->sync_to = to->engine->id; - __entry->seqno = from->fence.seqno; + __entry->seqno = from->global_seqno; ), TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u", @@ -489,7 +489,7 @@ TRACE_EVENT(i915_gem_ring_dispatch, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->flags = flags; dma_fence_enable_sw_signaling(&req->fence); ), @@ -534,7 +534,7 @@ DECLARE_EVENT_CLASS(i915_gem_request, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; ), TP_printk("dev=%u, ring=%u, seqno=%u", @@ -596,7 +596,7 @@ TRACE_EVENT(i915_gem_request_wait_begin, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->blocking = mutex_is_locked(&req->i915->drm.struct_mutex); ), diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index 56efcc507ea2..0d5def0d2dfe 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -504,9 +504,11 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) /* locked by dma_fence_enable_sw_signaling() */ assert_spin_locked(&request->lock); + if (!request->global_seqno) + return; request->signaling.wait.tsk = b->signaler; - request->signaling.wait.seqno = request->fence.seqno; + request->signaling.wait.seqno = request->global_seqno; i915_gem_request_get(request); spin_lock(&b->lock); @@ -530,8 +532,8 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) p = &b->signals.rb_node; while (*p) { parent = *p; - if (i915_seqno_passed(request->fence.seqno, - to_signaler(parent)->fence.seqno)) { + if (i915_seqno_passed(request->global_seqno, + to_signaler(parent)->global_seqno)) { p = &parent->rb_right; first = false; } else { diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index cb30549dfd40..e0a9bf81774b 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -1584,7 +1584,7 @@ static int gen8_emit_request(struct drm_i915_gem_request *request) intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT); intel_ring_emit(ring, 0); - intel_ring_emit(ring, request->fence.seqno); + intel_ring_emit(ring, request->global_seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_emit(ring, MI_NOOP); return intel_logical_ring_advance(request); @@ -1613,7 +1613,7 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request) PIPE_CONTROL_QW_WRITE)); intel_ring_emit(ring, intel_hws_seqno_address(request->engine)); intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(request)); + intel_ring_emit(ring, request->global_seqno); /* We're thrashing one dword of HWS. */ intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_USER_INTERRUPT); diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index aaa46d9ffbc1..76c6b70303fb 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -1238,7 +1238,7 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *req) PIPE_CONTROL_CS_STALL); intel_ring_emit(ring, lower_32_bits(gtt_offset)); intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_SEMAPHORE_SIGNAL | @@ -1274,7 +1274,7 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *req) lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT); intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); intel_ring_emit(ring, MI_SEMAPHORE_SIGNAL | MI_SEMAPHORE_TARGET(waiter->hw_id)); @@ -1308,7 +1308,7 @@ static int gen6_signal(struct drm_i915_gem_request *req) if (i915_mmio_reg_valid(mbox_reg)) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit_reg(ring, mbox_reg); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); } } @@ -1339,7 +1339,7 @@ static int i9xx_emit_request(struct drm_i915_gem_request *req) intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_advance(ring); @@ -1389,7 +1389,7 @@ static int gen8_render_emit_request(struct drm_i915_gem_request *req) PIPE_CONTROL_QW_WRITE)); intel_ring_emit(ring, intel_hws_seqno_address(engine)); intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(req)); + intel_ring_emit(ring, req->global_seqno); /* We're thrashing one dword of HWS. */ intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_USER_INTERRUPT); @@ -1427,7 +1427,7 @@ gen8_ring_sync_to(struct drm_i915_gem_request *req, MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT | MI_SEMAPHORE_SAD_GTE_SDD); - intel_ring_emit(ring, signal->fence.seqno); + intel_ring_emit(ring, signal->global_seqno); intel_ring_emit(ring, lower_32_bits(offset)); intel_ring_emit(ring, upper_32_bits(offset)); intel_ring_advance(ring); @@ -1465,7 +1465,7 @@ gen6_ring_sync_to(struct drm_i915_gem_request *req, * seqno is >= the last seqno executed. However for hardware the * comparison is strictly greater than. */ - intel_ring_emit(ring, signal->fence.seqno - 1); + intel_ring_emit(ring, signal->global_seqno - 1); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); -- cgit v1.2.3 From 85e17f5974b357bc4a127be09de71b430be265e0 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:53 +0100 Subject: drm/i915: Move the global sync optimisation to the timeline Currently we try to reduce the number of synchronisations (now the number of requests we need to wait upon) by noting that if we have earlier waited upon a request, all subsequent requests in the timeline will be after the wait. This only applies to requests in this timeline, as other timelines will not be ordered by that waiter. Signed-off-by: Chris Wilson Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-30-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 9 ------ drivers/gpu/drm/i915/i915_drv.h | 1 - drivers/gpu/drm/i915/i915_gem_request.c | 29 +++++++++++-------- drivers/gpu/drm/i915/i915_gem_timeline.h | 1 + drivers/gpu/drm/i915/i915_gpu_error.c | 48 +++++++++++++++++++------------- drivers/gpu/drm/i915/intel_engine_cs.c | 2 -- drivers/gpu/drm/i915/intel_ringbuffer.c | 3 -- drivers/gpu/drm/i915/intel_ringbuffer.h | 23 --------------- 8 files changed, 47 insertions(+), 69 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 90bc4a89e0d5..4655227eb9d9 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -3347,15 +3347,6 @@ static int i915_semaphore_status(struct seq_file *m, void *unused) seq_putc(m, '\n'); } - seq_puts(m, "\nSync seqno:\n"); - for_each_engine(engine, dev_priv, id) { - for (j = 0; j < num_rings; j++) - seq_printf(m, " 0x%08x ", - engine->semaphore.sync_seqno[j]); - seq_putc(m, '\n'); - } - seq_putc(m, '\n'); - intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); return 0; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 217674bb1495..6bf40276ace6 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -802,7 +802,6 @@ struct drm_i915_error_state { u32 cpu_ring_tail; u32 last_seqno; - u32 semaphore_seqno[I915_NUM_ENGINES - 1]; /* Register state */ u32 start; diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index 06daa4d203a7..9c34a4c540b5 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -238,35 +238,41 @@ static int i915_gem_check_wedge(struct drm_i915_private *dev_priv) return 0; } -static int i915_gem_init_global_seqno(struct drm_i915_private *dev_priv, - u32 seqno) +static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno) { - struct i915_gem_timeline *timeline = &dev_priv->gt.global_timeline; + struct i915_gem_timeline *timeline = &i915->gt.global_timeline; struct intel_engine_cs *engine; enum intel_engine_id id; int ret; /* Carefully retire all requests without writing to the rings */ - ret = i915_gem_wait_for_idle(dev_priv, + ret = i915_gem_wait_for_idle(i915, I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED); if (ret) return ret; - i915_gem_retire_requests(dev_priv); + i915_gem_retire_requests(i915); /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ if (!i915_seqno_passed(seqno, timeline->next_seqno)) { - while (intel_kick_waiters(dev_priv) || - intel_kick_signalers(dev_priv)) + while (intel_kick_waiters(i915) || intel_kick_signalers(i915)) yield(); yield(); } /* Finally reset hw state */ - for_each_engine(engine, dev_priv, id) + for_each_engine(engine, i915, id) intel_engine_init_global_seqno(engine, seqno); + list_for_each_entry(timeline, &i915->gt.timelines, link) { + for_each_engine(engine, i915, id) { + struct intel_timeline *tl = &timeline->engine[id]; + + memset(tl->sync_seqno, 0, sizeof(tl->sync_seqno)); + } + } + return 0; } @@ -462,7 +468,7 @@ static int i915_gem_request_await_request(struct drm_i915_gem_request *to, struct drm_i915_gem_request *from) { - int idx, ret; + int ret; GEM_BUG_ON(to == from); @@ -483,8 +489,7 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret < 0 ? ret : 0; } - idx = intel_engine_sync_index(from->engine, to->engine); - if (from->global_seqno <= from->engine->semaphore.sync_seqno[idx]) + if (from->global_seqno <= to->timeline->sync_seqno[from->engine->id]) return 0; trace_i915_gem_ring_sync_to(to, from); @@ -502,7 +507,7 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret; } - from->engine->semaphore.sync_seqno[idx] = from->global_seqno; + to->timeline->sync_seqno[from->engine->id] = from->global_seqno; return 0; } diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h index bfdf0331cc50..767b23914ec5 100644 --- a/drivers/gpu/drm/i915/i915_gem_timeline.h +++ b/drivers/gpu/drm/i915/i915_gem_timeline.h @@ -48,6 +48,7 @@ struct intel_timeline { * struct_mutex. */ struct i915_gem_active last_request; + u32 sync_seqno[I915_NUM_ENGINES]; struct i915_gem_timeline *common; }; diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 9aa197ca6210..ef3698120d92 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -415,17 +415,13 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, if (INTEL_GEN(m->i915) >= 6) { err_printf(m, " RC PSMI: 0x%08x\n", ee->rc_psmi); err_printf(m, " FAULT_REG: 0x%08x\n", ee->fault_reg); - err_printf(m, " SYNC_0: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[0], - ee->semaphore_seqno[0]); - err_printf(m, " SYNC_1: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[1], - ee->semaphore_seqno[1]); - if (HAS_VEBOX(m->i915)) { - err_printf(m, " SYNC_2: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[2], - ee->semaphore_seqno[2]); - } + err_printf(m, " SYNC_0: 0x%08x\n", + ee->semaphore_mboxes[0]); + err_printf(m, " SYNC_1: 0x%08x\n", + ee->semaphore_mboxes[1]); + if (HAS_VEBOX(m->i915)) + err_printf(m, " SYNC_2: 0x%08x\n", + ee->semaphore_mboxes[2]); } if (USES_PPGTT(m->i915)) { err_printf(m, " GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode); @@ -972,6 +968,26 @@ static void i915_gem_record_fences(struct drm_i915_private *dev_priv, } } +static inline u32 +gen8_engine_sync_index(struct intel_engine_cs *engine, + struct intel_engine_cs *other) +{ + int idx; + + /* + * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2; + * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs; + * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs; + * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs; + * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs; + */ + + idx = (other - engine) - 1; + if (idx < 0) + idx += I915_NUM_ENGINES; + + return idx; +} static void gen8_record_semaphore_state(struct drm_i915_error_state *error, struct intel_engine_cs *engine, @@ -995,10 +1011,9 @@ static void gen8_record_semaphore_state(struct drm_i915_error_state *error, signal_offset = (GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4; tmp = error->semaphore->pages[0]; - idx = intel_engine_sync_index(engine, to); + idx = gen8_engine_sync_index(engine, to); ee->semaphore_mboxes[idx] = tmp[signal_offset]; - ee->semaphore_seqno[idx] = engine->semaphore.sync_seqno[idx]; } } @@ -1009,14 +1024,9 @@ static void gen6_record_semaphore_state(struct intel_engine_cs *engine, ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base)); ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base)); - ee->semaphore_seqno[0] = engine->semaphore.sync_seqno[0]; - ee->semaphore_seqno[1] = engine->semaphore.sync_seqno[1]; - - if (HAS_VEBOX(dev_priv)) { + if (HAS_VEBOX(dev_priv)) ee->semaphore_mboxes[2] = I915_READ(RING_SYNC_2(engine->mmio_base)); - ee->semaphore_seqno[2] = engine->semaphore.sync_seqno[2]; - } } static void error_record_engine_waiters(struct intel_engine_cs *engine, diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c index 6a3105512d18..94de3d66733d 100644 --- a/drivers/gpu/drm/i915/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/intel_engine_cs.c @@ -204,8 +204,6 @@ void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno) I915_NUM_ENGINES * gen8_semaphore_seqno_size); kunmap(page); } - memset(engine->semaphore.sync_seqno, 0, - sizeof(engine->semaphore.sync_seqno)); intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); if (engine->irq_seqno_barrier) diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 16244775b9d1..188fdec5fa6b 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -2003,9 +2003,6 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine) intel_engine_setup_common(engine); - memset(engine->semaphore.sync_seqno, 0, - sizeof(engine->semaphore.sync_seqno)); - ret = intel_engine_init_common(engine); if (ret) goto error; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 9d228bee3511..891629caab6c 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -314,8 +314,6 @@ struct intel_engine_cs { * ie. transpose of f(x, y) */ struct { - u32 sync_seqno[I915_NUM_ENGINES-1]; - union { #define GEN6_SEMAPHORE_LAST VECS_HW #define GEN6_NUM_SEMAPHORES (GEN6_SEMAPHORE_LAST + 1) @@ -385,27 +383,6 @@ intel_engine_flag(const struct intel_engine_cs *engine) return 1 << engine->id; } -static inline u32 -intel_engine_sync_index(struct intel_engine_cs *engine, - struct intel_engine_cs *other) -{ - int idx; - - /* - * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2; - * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs; - * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs; - * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs; - * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs; - */ - - idx = (other->id - engine->id) - 1; - if (idx < 0) - idx += I915_NUM_ENGINES; - - return idx; -} - static inline void intel_flush_status_page(struct intel_engine_cs *engine, int reg) { -- cgit v1.2.3 From 562f5d45508bc376cf537d414cd944cff5e453f7 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:54 +0100 Subject: drm/i915: Create a unique name for the context This will be used for communicating issues with this context to userspace, so we want to identify the parent process and the individual context. Note that the name isn't quite unique, it makes the presumption of there only being a single device fd per process. Signed-off-by: Chris Wilson Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-31-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 11 ++--------- drivers/gpu/drm/i915/i915_drv.h | 1 + drivers/gpu/drm/i915/i915_gem_context.c | 23 ++++++++++++++++++----- 3 files changed, 21 insertions(+), 14 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 4655227eb9d9..f8604a09c101 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -631,17 +631,10 @@ static void print_request(struct seq_file *m, struct drm_i915_gem_request *rq, const char *prefix) { - struct pid *pid = rq->ctx->pid; - struct task_struct *task; - - rcu_read_lock(); - task = pid ? pid_task(pid, PIDTYPE_PID) : NULL; - seq_printf(m, "%s%x [%x:%x] @ %d: %s [%d]\n", prefix, + seq_printf(m, "%s%x [%x:%x] @ %d: %s\n", prefix, rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno, jiffies_to_msecs(jiffies - rq->emitted_jiffies), - task ? task->comm : "", - task ? task->pid : -1); - rcu_read_unlock(); + rq->timeline->common->name); } static int i915_gem_request_info(struct seq_file *m, void *data) diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 6bf40276ace6..8b987f772d47 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -937,6 +937,7 @@ struct i915_gem_context { struct drm_i915_file_private *file_priv; struct i915_hw_ppgtt *ppgtt; struct pid *pid; + const char *name; struct i915_ctx_hang_stats hang_stats; diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index a2acb8bb5f34..d3118db244c4 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -158,6 +158,7 @@ void i915_gem_context_free(struct kref *ctx_ref) __i915_gem_object_release_unless_active(ce->state->obj); } + kfree(ctx->name); put_pid(ctx->pid); list_del(&ctx->link); @@ -303,19 +304,28 @@ __create_hw_context(struct drm_device *dev, } /* Default context will never have a file_priv */ - if (file_priv != NULL) { + ret = DEFAULT_CONTEXT_HANDLE; + if (file_priv) { ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL); if (ret < 0) goto err_out; - } else - ret = DEFAULT_CONTEXT_HANDLE; + } + ctx->user_handle = ret; ctx->file_priv = file_priv; - if (file_priv) + if (file_priv) { ctx->pid = get_task_pid(current, PIDTYPE_PID); + ctx->name = kasprintf(GFP_KERNEL, "%s[%d]/%x", + current->comm, + pid_nr(ctx->pid), + ctx->user_handle); + if (!ctx->name) { + ret = -ENOMEM; + goto err_pid; + } + } - ctx->user_handle = ret; /* NB: Mark all slices as needing a remap so that when the context first * loads it will restore whatever remap state already exists. If there * is no remap info, it will be a NOP. */ @@ -329,6 +339,9 @@ __create_hw_context(struct drm_device *dev, return ctx; +err_pid: + put_pid(ctx->pid); + idr_remove(&file_priv->context_idr, ctx->user_handle); err_out: context_close(ctx); return ERR_PTR(ret); -- cgit v1.2.3 From f6168e330438a264123d2e0b502526f06594bb51 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:55 +0100 Subject: drm/i915: Convert breadcrumbs spinlock to be irqsafe The breadcrumbs are about to be used from within IRQ context sections (e.g. nouveau signals a fence from an interrupt handler causing us to submit a new request) and/or from bottom-half tasklets (i.e. intel_lrc_irq_handler), therefore we need to employ the irqsafe spinlock variants. For example, deferring the request submission to the intel_lrc_irq_handler generates this trace: [ 66.388639] ================================= [ 66.388650] [ INFO: inconsistent lock state ] [ 66.388663] 4.9.0-rc2+ #56 Not tainted [ 66.388672] --------------------------------- [ 66.388682] inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage. [ 66.388695] swapper/1/0 [HC0[0]:SC1[1]:HE0:SE0] takes: [ 66.388706] (&(&b->lock)->rlock){+.?...} , at: [] intel_engine_enable_signaling+0x78/0x150 [ 66.388761] {SOFTIRQ-ON-W} state was registered at: [ 66.388772] [ 66.388783] [] __lock_acquire+0x682/0x1870 [ 66.388795] [ 66.388803] [] lock_acquire+0x6c/0xb0 [ 66.388814] [ 66.388824] [] _raw_spin_lock+0x2a/0x40 [ 66.388835] [ 66.388845] [] intel_engine_reset_breadcrumbs+0x21/0xb0 [ 66.388857] [ 66.388866] [] gen8_init_common_ring+0x67/0x100 [ 66.388878] [ 66.388887] [] gen8_init_render_ring+0x12/0x60 [ 66.388903] [ 66.388912] [] i915_gem_init_hw+0xf7/0x2a0 [ 66.388927] [ 66.388936] [] i915_gem_init+0xbb/0xf0 [ 66.388950] [ 66.388959] [] i915_driver_load+0x7e0/0x1330 [ 66.388978] [ 66.388988] [] i915_pci_probe+0x28/0x40 [ 66.389003] [ 66.389013] [] pci_device_probe+0x8b/0xf0 [ 66.389028] [ 66.389037] [] driver_probe_device+0x21e/0x430 [ 66.389056] [ 66.389065] [] __driver_attach+0xde/0xe0 [ 66.389080] [ 66.389090] [] bus_for_each_dev+0x5d/0x90 [ 66.389105] [ 66.389113] [] driver_attach+0x19/0x20 [ 66.389134] [ 66.389144] [] bus_add_driver+0x15d/0x260 [ 66.389159] [ 66.389168] [] driver_register+0x5b/0xd0 [ 66.389183] [ 66.389281] [] __pci_register_driver+0x5b/0x60 [ 66.389301] [ 66.389312] [] i915_init+0x3e/0x45 [ 66.389326] [ 66.389336] [] do_one_initcall+0x8b/0x118 [ 66.389350] [ 66.389359] [] kernel_init_freeable+0x1b3/0x23b [ 66.389378] [ 66.389387] [] kernel_init+0x9/0x100 [ 66.389402] [ 66.389411] [] ret_from_fork+0x27/0x40 [ 66.389426] irq event stamp: 315865 [ 66.389438] hardirqs last enabled at (315864): [] _raw_spin_unlock_irqrestore+0x31/0x50 [ 66.389469] hardirqs last disabled at (315865): [] _raw_spin_lock_irqsave+0x13/0x50 [ 66.389499] softirqs last enabled at (315818): [] _local_bh_enable+0x1c/0x50 [ 66.389530] softirqs last disabled at (315819): [] irq_exit+0xbe/0xd0 [ 66.389559] [ 66.389559] other info that might help us debug this: [ 66.389580] Possible unsafe locking scenario: [ 66.389580] [ 66.389598] CPU0 [ 66.389609] ---- [ 66.389620] lock(&(&b->lock)->rlock); [ 66.389650] [ 66.389661] lock(&(&b->lock)->rlock); [ 66.389690] [ 66.389690] *** DEADLOCK *** [ 66.389690] [ 66.389715] 2 locks held by swapper/1/0: [ 66.389728] #0: (&(&tl->lock)->rlock){..-...}, at: [] intel_lrc_irq_handler+0x201/0x3c0 [ 66.389785] #1: (&(&req->lock)->rlock/1){..-...}, at: [] __i915_gem_request_submit+0x8f/0x170 [ 66.389854] [ 66.389854] stack backtrace: [ 66.389959] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 4.9.0-rc2+ #56 [ 66.389976] Hardware name: / , BIOS PYBSWCEL.86A.0027.2015.0507.1758 05/07/2015 [ 66.389999] ffff88027fd03c58 ffffffff812beae5 ffff88027696e680 ffffffff822afe20 [ 66.390036] ffff88027fd03ca8 ffffffff810bb420 0000000000000001 0000000000000000 [ 66.390070] 0000000000000000 0000000000000006 0000000000000004 ffff88027696ee10 [ 66.390104] Call Trace: [ 66.390117] [ 66.390128] [] dump_stack+0x68/0x93 [ 66.390147] [] print_usage_bug+0x1d0/0x1e0 [ 66.390164] [] mark_lock+0x470/0x4f0 [ 66.390181] [] ? print_shortest_lock_dependencies+0x1b0/0x1b0 [ 66.390203] [] __lock_acquire+0x59d/0x1870 [ 66.390221] [] lock_acquire+0x6c/0xb0 [ 66.390237] [] ? lock_acquire+0x6c/0xb0 [ 66.390255] [] ? intel_engine_enable_signaling+0x78/0x150 [ 66.390273] [] _raw_spin_lock+0x2a/0x40 [ 66.390291] [] ? intel_engine_enable_signaling+0x78/0x150 [ 66.390309] [] intel_engine_enable_signaling+0x78/0x150 [ 66.390327] [] __i915_gem_request_submit+0x150/0x170 [ 66.390345] [] intel_lrc_irq_handler+0x28b/0x3c0 [ 66.390363] [] tasklet_action+0x57/0xc0 [ 66.390380] [] __do_softirq+0x119/0x240 [ 66.390396] [] irq_exit+0xbe/0xd0 [ 66.390414] [] do_IRQ+0x65/0x110 [ 66.390431] [] common_interrupt+0x86/0x86 [ 66.390446] [ 66.390457] [] ? cpuidle_enter_state+0x151/0x200 [ 66.390480] [] cpuidle_enter+0x12/0x20 [ 66.390498] [] call_cpuidle+0x1e/0x40 [ 66.390516] [] cpu_startup_entry+0x10e/0x1f0 [ 66.390534] [] start_secondary+0x103/0x130 (This is split out of the defer global seqno allocation patch due to realisation that we need a more complete conversion if we want to defer request submission even further.) v2: lockdep was warning about mixed SOFTIRQ contexts not HARDIRQ contexts so we only need to use spin_lock_bh and not disable interrupts. v3: We need full irq protection as we may be called from a third party interrupt handler (via fences). Signed-off-by: Chris Wilson Cc: Joonas Lahtinen Cc: Tvrtko Ursulin Reviewed-by: Joonas Lahtinen Reviewed-by: Tvrtko Ursulin Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-32-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 12 +++++------ drivers/gpu/drm/i915/i915_gpu_error.c | 8 ++++---- drivers/gpu/drm/i915/intel_breadcrumbs.c | 35 ++++++++++++++++++++------------ drivers/gpu/drm/i915/intel_ringbuffer.h | 2 +- 4 files changed, 33 insertions(+), 24 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index f8604a09c101..1723a1f5b20e 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -683,14 +683,14 @@ static void i915_ring_seqno_info(struct seq_file *m, seq_printf(m, "Current sequence (%s): %x\n", engine->name, intel_engine_get_seqno(engine)); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "Waiting (%s): %s [%d] on %x\n", engine->name, w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static int i915_gem_seqno_info(struct seq_file *m, void *data) @@ -1355,14 +1355,14 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) yesno(intel_engine_has_waiter(engine)), yesno(test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings))); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "\t%s [%d] waiting for %x\n", w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n", (long long)engine->hangcheck.acthd, @@ -3265,14 +3265,14 @@ static int i915_engine_info(struct seq_file *m, void *unused) I915_READ(RING_PP_DIR_DCLV(engine))); } - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "\t%s [%d] waiting for %x\n", w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); seq_puts(m, "\n"); } diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index ef3698120d92..7ba40487e345 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1043,7 +1043,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (RB_EMPTY_ROOT(&b->waiters)) return; - if (!spin_trylock(&b->lock)) { + if (!spin_trylock_irq(&b->lock)) { ee->waiters = ERR_PTR(-EDEADLK); return; } @@ -1051,7 +1051,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, count = 0; for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb)) count++; - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); waiter = NULL; if (count) @@ -1061,7 +1061,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (!waiter) return; - if (!spin_trylock(&b->lock)) { + if (!spin_trylock_irq(&b->lock)) { kfree(waiter); ee->waiters = ERR_PTR(-EDEADLK); return; @@ -1079,7 +1079,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (++ee->num_waiters == count) break; } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static void error_record_engine_registers(struct drm_i915_error_state *error, diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index 0d5def0d2dfe..c410d3d6465f 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -83,16 +83,18 @@ static void irq_enable(struct intel_engine_cs *engine) */ engine->breadcrumbs.irq_posted = true; - spin_lock_irq(&engine->i915->irq_lock); + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); engine->irq_enable(engine); - spin_unlock_irq(&engine->i915->irq_lock); + spin_unlock(&engine->i915->irq_lock); } static void irq_disable(struct intel_engine_cs *engine) { - spin_lock_irq(&engine->i915->irq_lock); + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); engine->irq_disable(engine); - spin_unlock_irq(&engine->i915->irq_lock); + spin_unlock(&engine->i915->irq_lock); engine->breadcrumbs.irq_posted = false; } @@ -293,9 +295,9 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine, struct intel_breadcrumbs *b = &engine->breadcrumbs; bool first; - spin_lock(&b->lock); + spin_lock_irq(&b->lock); first = __intel_engine_add_wait(engine, wait); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); return first; } @@ -326,7 +328,7 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine, if (RB_EMPTY_NODE(&wait->node)) return; - spin_lock(&b->lock); + spin_lock_irq(&b->lock); if (RB_EMPTY_NODE(&wait->node)) goto out_unlock; @@ -400,7 +402,7 @@ out_unlock: GEM_BUG_ON(rb_first(&b->waiters) != (b->first_wait ? &b->first_wait->node : NULL)); GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh) ^ RB_EMPTY_ROOT(&b->waiters)); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static bool signal_complete(struct drm_i915_gem_request *request) @@ -473,14 +475,14 @@ static int intel_breadcrumbs_signaler(void *arg) * we just completed - so double check we are still * the oldest before picking the next one. */ - spin_lock(&b->lock); + spin_lock_irq(&b->lock); if (request == b->first_signal) { struct rb_node *rb = rb_next(&request->signaling.node); b->first_signal = rb ? to_signaler(rb) : NULL; } rb_erase(&request->signaling.node, &b->signals); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); i915_gem_request_put(request); } else { @@ -502,7 +504,14 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) struct rb_node *parent, **p; bool first, wakeup; - /* locked by dma_fence_enable_sw_signaling() */ + /* Note that we may be called from an interrupt handler on another + * device (e.g. nouveau signaling a fence completion causing us + * to submit a request, and so enable signaling). As such, + * we need to make sure that all other users of b->lock protect + * against interrupts, i.e. use spin_lock_irqsave. + */ + + /* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */ assert_spin_locked(&request->lock); if (!request->global_seqno) return; @@ -594,7 +603,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) struct intel_breadcrumbs *b = &engine->breadcrumbs; cancel_fake_irq(engine); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); __intel_breadcrumbs_disable_irq(b); if (intel_engine_has_waiter(engine)) { @@ -607,7 +616,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) irq_disable(engine); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 891629caab6c..d16c74ae8f54 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -207,7 +207,7 @@ struct intel_engine_cs { struct task_struct __rcu *irq_seqno_bh; /* bh for interrupts */ bool irq_posted; - spinlock_t lock; /* protects the lists of requests */ + spinlock_t lock; /* protects the lists of requests; irqsafe */ struct rb_root waiters; /* sorted by retirement, priority */ struct rb_root signals; /* sorted by retirement */ struct intel_wait *first_wait; /* oldest waiter by retirement */ -- cgit v1.2.3 From 28176ef4cfa510e5f1498bbf39ff1e4afd0b085d Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 28 Oct 2016 13:58:56 +0100 Subject: drm/i915: Reserve space in the global seqno during request allocation A restriction on our global seqno is that they cannot wrap, and that we cannot use the value 0. This allows us to detect when a request has not yet been submitted, its global seqno is still 0, and ensures that hardware semaphores are monotonic as required by older hardware. To meet these restrictions when we defer the assignment of the global seqno, we must check that we have an available slot in the global seqno space during request construction. If that test fails, we wait for all requests to be completed and reset the hardware back to 0. Signed-off-by: Chris Wilson Reviewed-by: Joonas Lahtinen Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-33-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 10 ++-- drivers/gpu/drm/i915/i915_drv.h | 2 +- drivers/gpu/drm/i915/i915_gem.c | 7 ++- drivers/gpu/drm/i915/i915_gem_request.c | 86 +++++++++++++++++--------------- drivers/gpu/drm/i915/i915_gem_timeline.h | 2 +- 5 files changed, 55 insertions(+), 52 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 1723a1f5b20e..9bef6f55f99d 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -552,7 +552,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) seq_printf(m, "Flip queued on %s at seqno %x, next seqno %x [current breadcrumb %x], completed? %d\n", engine->name, i915_gem_request_get_seqno(work->flip_queued_req), - dev_priv->gt.global_timeline.next_seqno, + atomic_read(&dev_priv->gt.global_timeline.next_seqno), intel_engine_get_seqno(engine), i915_gem_request_completed(work->flip_queued_req)); } else @@ -1046,7 +1046,7 @@ i915_next_seqno_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - *val = READ_ONCE(dev_priv->gt.global_timeline.next_seqno); + *val = atomic_read(&dev_priv->gt.global_timeline.next_seqno); return 0; } @@ -2277,8 +2277,8 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) struct drm_file *file; seq_printf(m, "RPS enabled? %d\n", dev_priv->rps.enabled); - seq_printf(m, "GPU busy? %s [%x]\n", - yesno(dev_priv->gt.awake), dev_priv->gt.active_engines); + seq_printf(m, "GPU busy? %s [%d requests]\n", + yesno(dev_priv->gt.awake), dev_priv->gt.active_requests); seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv)); seq_printf(m, "Frequency requested %d\n", intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq)); @@ -2313,7 +2313,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) if (INTEL_GEN(dev_priv) >= 6 && dev_priv->rps.enabled && - dev_priv->gt.active_engines) { + dev_priv->gt.active_requests) { u32 rpup, rpupei; u32 rpdown, rpdownei; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 8b987f772d47..eacb144af29e 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -2092,6 +2092,7 @@ struct drm_i915_private { struct list_head timelines; struct i915_gem_timeline global_timeline; + u32 active_requests; /** * Is the GPU currently considered idle, or busy executing @@ -2100,7 +2101,6 @@ struct drm_i915_private { * In order to reduce the effect on performance, there * is a slight delay before we do so. */ - unsigned int active_engines; bool awake; /** diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index f4cfb88bd804..8a5d20715e5f 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -2688,8 +2688,6 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) memset(engine->execlist_port, 0, sizeof(engine->execlist_port)); spin_unlock(&engine->execlist_lock); } - - engine->i915->gt.active_engines &= ~intel_engine_flag(engine); } void i915_gem_set_wedged(struct drm_i915_private *dev_priv) @@ -2746,7 +2744,7 @@ i915_gem_idle_work_handler(struct work_struct *work) if (!READ_ONCE(dev_priv->gt.awake)) return; - if (READ_ONCE(dev_priv->gt.active_engines)) + if (READ_ONCE(dev_priv->gt.active_requests)) return; rearm_hangcheck = @@ -2760,7 +2758,7 @@ i915_gem_idle_work_handler(struct work_struct *work) goto out_rearm; } - if (dev_priv->gt.active_engines) + if (dev_priv->gt.active_requests) goto out_unlock; for_each_engine(engine, dev_priv, id) @@ -4399,6 +4397,7 @@ int i915_gem_suspend(struct drm_device *dev) goto err; i915_gem_retire_requests(dev_priv); + GEM_BUG_ON(dev_priv->gt.active_requests); assert_kernel_context_is_current(dev_priv); i915_gem_context_lost(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index 9c34a4c540b5..9b22f66464f0 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -159,6 +159,7 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) */ list_del(&request->ring_link); request->ring->last_retired_head = request->postfix; + request->i915->gt.active_requests--; /* Walk through the active list, calling retire on each. This allows * objects to track their GPU activity and mark themselves as idle @@ -253,13 +254,15 @@ static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno) return ret; i915_gem_retire_requests(i915); + GEM_BUG_ON(i915->gt.active_requests > 1); /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ - if (!i915_seqno_passed(seqno, timeline->next_seqno)) { + if (!i915_seqno_passed(seqno, atomic_read(&timeline->next_seqno))) { while (intel_kick_waiters(i915) || intel_kick_signalers(i915)) yield(); yield(); } + atomic_set(&timeline->next_seqno, seqno); /* Finally reset hw state */ for_each_engine(engine, i915, id) @@ -279,7 +282,6 @@ static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno) int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno) { struct drm_i915_private *dev_priv = to_i915(dev); - int ret; lockdep_assert_held(&dev_priv->drm.struct_mutex); @@ -289,34 +291,33 @@ int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno) /* HWS page needs to be set less than what we * will inject to ring */ - ret = i915_gem_init_global_seqno(dev_priv, seqno - 1); - if (ret) - return ret; - - dev_priv->gt.global_timeline.next_seqno = seqno; - return 0; + return i915_gem_init_global_seqno(dev_priv, seqno - 1); } -static int i915_gem_get_global_seqno(struct drm_i915_private *dev_priv, - u32 *seqno) +static int reserve_global_seqno(struct drm_i915_private *i915) { - struct i915_gem_timeline *tl = &dev_priv->gt.global_timeline; - - /* reserve 0 for non-seqno */ - if (unlikely(tl->next_seqno == 0)) { - int ret; + u32 active_requests = ++i915->gt.active_requests; + u32 next_seqno = atomic_read(&i915->gt.global_timeline.next_seqno); + int ret; - ret = i915_gem_init_global_seqno(dev_priv, 0); - if (ret) - return ret; + /* Reservation is fine until we need to wrap around */ + if (likely(next_seqno + active_requests > next_seqno)) + return 0; - tl->next_seqno = 1; + ret = i915_gem_init_global_seqno(i915, 0); + if (ret) { + i915->gt.active_requests--; + return ret; } - *seqno = tl->next_seqno++; return 0; } +static u32 timeline_get_seqno(struct i915_gem_timeline *tl) +{ + return atomic_inc_return(&tl->next_seqno); +} + static int __i915_sw_fence_call submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) { @@ -356,9 +357,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, { struct drm_i915_private *dev_priv = engine->i915; struct drm_i915_gem_request *req; - u32 seqno; int ret; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex * and restart. @@ -367,6 +369,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, if (ret) return ERR_PTR(ret); + ret = reserve_global_seqno(dev_priv); + if (ret) + return ERR_PTR(ret); + /* Move the oldest request to the slab-cache (if not in use!) */ req = list_first_entry_or_null(&engine->timeline->requests, typeof(*req), link); @@ -402,12 +408,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * Do not use kmem_cache_zalloc() here! */ req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL); - if (!req) - return ERR_PTR(-ENOMEM); - - ret = i915_gem_get_global_seqno(dev_priv, &seqno); - if (ret) - goto err; + if (!req) { + ret = -ENOMEM; + goto err_unreserve; + } req->timeline = engine->timeline; @@ -416,14 +420,14 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, &i915_fence_ops, &req->lock, req->timeline->fence_context, - seqno); + timeline_get_seqno(req->timeline->common)); i915_sw_fence_init(&req->submit, submit_notify); INIT_LIST_HEAD(&req->active_list); req->i915 = dev_priv; req->engine = engine; - req->global_seqno = seqno; + req->global_seqno = req->fence.seqno; req->ctx = i915_gem_context_get(ctx); /* No zalloc, must clear what we need by hand */ @@ -459,8 +463,9 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, err_ctx: i915_gem_context_put(ctx); -err: kmem_cache_free(dev_priv->requests, req); +err_unreserve: + dev_priv->gt.active_requests--; return ERR_PTR(ret); } @@ -624,7 +629,6 @@ static void i915_gem_mark_busy(const struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; - dev_priv->gt.active_engines |= intel_engine_flag(engine); if (dev_priv->gt.awake) return; @@ -700,6 +704,9 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) i915_sw_fence_await_sw_fence(&request->submit, &prev->submit, &request->submitq); + GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno, + request->fence.seqno)); + request->emitted_jiffies = jiffies; request->previous_seqno = timeline->last_pending_seqno; timeline->last_pending_seqno = request->fence.seqno; @@ -962,38 +969,35 @@ complete: return timeout; } -static bool engine_retire_requests(struct intel_engine_cs *engine) +static void engine_retire_requests(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request, *next; list_for_each_entry_safe(request, next, &engine->timeline->requests, link) { if (!i915_gem_request_completed(request)) - return false; + return; i915_gem_request_retire(request); } - - return true; } void i915_gem_retire_requests(struct drm_i915_private *dev_priv) { struct intel_engine_cs *engine; - unsigned int tmp; + enum intel_engine_id id; lockdep_assert_held(&dev_priv->drm.struct_mutex); - if (dev_priv->gt.active_engines == 0) + if (!dev_priv->gt.active_requests) return; GEM_BUG_ON(!dev_priv->gt.awake); - for_each_engine_masked(engine, dev_priv, dev_priv->gt.active_engines, tmp) - if (engine_retire_requests(engine)) - dev_priv->gt.active_engines &= ~intel_engine_flag(engine); + for_each_engine(engine, dev_priv, id) + engine_retire_requests(engine); - if (dev_priv->gt.active_engines == 0) + if (!dev_priv->gt.active_requests) queue_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, msecs_to_jiffies(100)); diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h index 767b23914ec5..18e603980dd9 100644 --- a/drivers/gpu/drm/i915/i915_gem_timeline.h +++ b/drivers/gpu/drm/i915/i915_gem_timeline.h @@ -55,7 +55,7 @@ struct intel_timeline { struct i915_gem_timeline { struct list_head link; - u32 next_seqno; + atomic_t next_seqno; struct drm_i915_private *i915; const char *name; -- cgit v1.2.3 From cb399eabc41ae0e67fe87fbebee4cb03ab1b5de9 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 1 Nov 2016 10:03:16 +0000 Subject: drm/i915: Avoid accessing request->timeline outside of its lifetime Whilst waiting on a request, we may do so without holding any locks or any guards beyond a reference to the request. In order to avoid taking locks within request deallocation, we drop references to its timeline (via the context and ppgtt) upon retirement. We should avoid chasing such pointers outside of their control, in particular we inspect the request->timeline to see if we may restore the RPS waitboost for a client. If we instead look at the engine->timeline, we will have similar behaviour on both full-ppgtt and !full-ppgtt systems and reduce the amount of reward we give towards stalling clients (i.e. only if the client stalls and the GPU is uncontended does it reclaim its boost). This restores behaviour back to pre-timelines, whilst fixing: [ 645.078485] BUG: KASAN: use-after-free in i915_gem_object_wait_fence+0x1ee/0x2e0 at addr ffff8802335643a0 [ 645.078577] Read of size 4 by task gem_exec_schedu/28408 [ 645.078638] CPU: 1 PID: 28408 Comm: gem_exec_schedu Not tainted 4.9.0-rc2+ #64 [ 645.078724] Hardware name: / , BIOS PYBSWCEL.86A.0027.2015.0507.1758 05/07/2015 [ 645.078816] ffff88022daef9a0 ffffffff8143d059 ffff880235402a80 ffff880233564200 [ 645.078998] ffff88022daef9c8 ffffffff81229c5c ffff88022daefa48 ffff880233564200 [ 645.079172] ffff880235402a80 ffff88022daefa38 ffffffff81229ef0 000000008110a796 [ 645.079345] Call Trace: [ 645.079404] [] dump_stack+0x68/0x9f [ 645.079467] [] kasan_object_err+0x1c/0x70 [ 645.079534] [] kasan_report_error+0x1f0/0x4b0 [ 645.079601] [] kasan_report+0x34/0x40 [ 645.079676] [] ? i915_gem_object_wait_fence+0x1ee/0x2e0 [ 645.079741] [] __asan_load4+0x61/0x80 [ 645.079807] [] i915_gem_object_wait_fence+0x1ee/0x2e0 [ 645.079876] [] i915_gem_object_wait+0x19f/0x590 [ 645.079944] [] ? i915_gem_object_wait_priority+0x500/0x500 [ 645.080016] [] ? debug_show_all_locks+0x1e0/0x1e0 [ 645.080084] [] ? check_chain_key+0x14c/0x210 [ 645.080157] [] ? __lock_is_held+0x46/0xc0 [ 645.080226] [] ? i915_gem_set_domain_ioctl+0x141/0x690 [ 645.080296] [] i915_gem_set_domain_ioctl+0x1a2/0x690 [ 645.080366] [] ? __might_fault+0x75/0xe0 [ 645.080433] [] drm_ioctl+0x327/0x640 [ 645.080508] [] ? i915_gem_obj_prepare_shmem_write+0x3a0/0x3a0 [ 645.080603] [] ? drm_ioctl_permit+0x120/0x120 [ 645.080670] [] ? check_chain_key+0x14c/0x210 [ 645.080738] [] do_vfs_ioctl+0x127/0xa20 [ 645.080804] [] ? do_mmap+0x47c/0x580 [ 645.080871] [] ? vm_mmap_pgoff+0x117/0x140 [ 645.080938] [] ? ioctl_preallocate+0x150/0x150 [ 645.081011] [] ? up_write+0x23/0x50 [ 645.081078] [] ? vm_mmap_pgoff+0x117/0x140 [ 645.081145] [] ? vma_is_stack_for_current+0x90/0x90 [ 645.081214] [] ? mark_held_locks+0x23/0xc0 [ 645.082030] [] ? __fget+0x168/0x250 [ 645.082106] [] ? entry_SYSCALL_64_fastpath+0x5/0xb1 [ 645.082176] [] ? __fget_light+0xa2/0xc0 [ 645.082242] [] SyS_ioctl+0x3c/0x70 [ 645.082309] [] entry_SYSCALL_64_fastpath+0x1c/0xb1 [ 645.082374] Object at ffff880233564200, in cache kmalloc-8192 size: 8192 [ 645.082431] Allocated: [ 645.082480] PID = 28408 [ 645.082535] [ 645.082566] [] save_stack_trace+0x16/0x20 [ 645.082623] [ 645.082656] [] save_stack+0x46/0xd0 [ 645.082716] [ 645.082756] [] kasan_kmalloc+0xad/0xe0 [ 645.082817] [ 645.082848] [] i915_ppgtt_create+0x52/0x220 [ 645.082908] [ 645.082941] [] i915_gem_create_context+0x396/0x560 [ 645.083027] [ 645.083059] [] i915_gem_context_create_ioctl+0x97/0xf0 [ 645.083152] [ 645.083183] [] drm_ioctl+0x327/0x640 [ 645.083243] [ 645.083274] [] do_vfs_ioctl+0x127/0xa20 [ 645.083334] [ 645.083372] [] SyS_ioctl+0x3c/0x70 [ 645.083432] [ 645.083464] [] entry_SYSCALL_64_fastpath+0x1c/0xb1 [ 645.083551] Freed: [ 645.083599] PID = 27629 [ 645.083648] [ 645.083676] [] save_stack_trace+0x16/0x20 [ 645.083738] [ 645.083770] [] save_stack+0x46/0xd0 [ 645.083830] [ 645.083862] [] kasan_slab_free+0x73/0xc0 [ 645.083922] [ 645.083961] [] kfree+0xa9/0x170 [ 645.084021] [ 645.084053] [] i915_ppgtt_release+0x100/0x180 [ 645.084139] [ 645.084171] [] i915_gem_context_free+0x1b4/0x230 [ 645.084257] [ 645.084288] [] intel_lr_context_unpin+0x192/0x230 [ 645.084380] [ 645.084413] [] i915_gem_request_retire+0x620/0x630 [ 645.084500] [ 645.085226] [] i915_gem_retire_requests+0x181/0x280 [ 645.085313] [ 645.085352] [] i915_gem_retire_work_handler+0xca/0xe0 [ 645.085440] [ 645.085471] [] process_one_work+0x4fb/0x920 [ 645.085532] [ 645.085562] [] worker_thread+0x8d/0x840 [ 645.085622] [ 645.085653] [] kthread+0x185/0x1b0 [ 645.085718] [ 645.085750] [] ret_from_fork+0x27/0x40 [ 645.085811] Memory state around the buggy address: [ 645.085869] ffff880233564280: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 645.085956] ffff880233564300: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 645.086053] >ffff880233564380: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 645.086138] ^ [ 645.086193] ffff880233564400: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 645.086283] ffff880233564480: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb v2: Add a comment to document the hint like nature of intel_engine_last_submit() Fixes: 73cb97010d4f ("drm/i915: Combine seqno + tracking into a global timeline struct") Fixes: 80b204bce8f2 ("drm/i915: Enable multiple timelines") Signed-off-by: Chris Wilson Cc: Joonas Lahtinen Reviewed-by: Tvrtko Ursulin Link: http://patchwork.freedesktop.org/patch/msgid/20161101100317.11129-1-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_debugfs.c | 7 +++---- drivers/gpu/drm/i915/i915_gem.c | 4 ++-- drivers/gpu/drm/i915/i915_gpu_error.c | 2 +- drivers/gpu/drm/i915/i915_irq.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.h | 12 ++++++++++++ 5 files changed, 19 insertions(+), 8 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 9bef6f55f99d..bf19192dcc3b 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -1348,9 +1348,8 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) seq_printf(m, "%s:\n", engine->name); seq_printf(m, "\tseqno = %x [current %x, last %x]\n", - engine->hangcheck.seqno, - seqno[id], - engine->timeline->last_submitted_seqno); + engine->hangcheck.seqno, seqno[id], + intel_engine_last_submit(engine)); seq_printf(m, "\twaiters? %s, fake irq active? %s\n", yesno(intel_engine_has_waiter(engine)), yesno(test_bit(engine->id, @@ -3167,7 +3166,7 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_printf(m, "%s\n", engine->name); seq_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [score %d]\n", intel_engine_get_seqno(engine), - engine->timeline->last_submitted_seqno, + intel_engine_last_submit(engine), engine->hangcheck.seqno, engine->hangcheck.score); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index b51274562e79..6c51b21565d6 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -371,7 +371,7 @@ out: if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq)) i915_gem_request_retire_upto(rq); - if (rps && rq->fence.seqno == rq->timeline->last_submitted_seqno) { + if (rps && rq->global_seqno == intel_engine_last_submit(rq->engine)) { /* The GPU is now idle and this client has stalled. * Since no other client has submitted a request in the * meantime, assume that this client is the only one @@ -2674,7 +2674,7 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) * reset it. */ intel_engine_init_global_seqno(engine, - engine->timeline->last_submitted_seqno); + intel_engine_last_submit(engine)); /* * Clear the execlists queue up before freeing the requests, as those diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 7ba40487e345..204093f3eaa5 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1120,7 +1120,7 @@ static void error_record_engine_registers(struct drm_i915_error_state *error, ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base)); ee->acthd = intel_engine_get_active_head(engine); ee->seqno = intel_engine_get_seqno(engine); - ee->last_seqno = engine->timeline->last_submitted_seqno; + ee->last_seqno = intel_engine_last_submit(engine); ee->start = I915_READ_START(engine); ee->head = I915_READ_HEAD(engine); ee->tail = I915_READ_TAIL(engine); diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 90d0905592f2..4dda2b1eefdb 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -3168,7 +3168,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work) acthd = intel_engine_get_active_head(engine); seqno = intel_engine_get_seqno(engine); - submit = READ_ONCE(engine->timeline->last_submitted_seqno); + submit = intel_engine_last_submit(engine); if (engine->hangcheck.seqno == seqno) { if (i915_seqno_passed(seqno, submit)) { diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 43f61aa24ed7..642b54692d0d 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -496,6 +496,18 @@ static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine) return intel_read_status_page(engine, I915_GEM_HWS_INDEX); } +static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine) +{ + /* We are only peeking at the tail of the submit queue (and not the + * queue itself) in order to gain a hint as to the current active + * state of the engine. Callers are not expected to be taking + * engine->timeline->lock, nor are they expected to be concerned + * wtih serialising this hint with anything, so document it as + * a hint and nothing more. + */ + return READ_ONCE(engine->timeline->last_submitted_seqno); +} + int init_workarounds_ring(struct intel_engine_cs *engine); void intel_engine_get_instdone(struct intel_engine_cs *engine, -- cgit v1.2.3 From e2af48c66b41fecbec5bc1b07640c2419a724c34 Mon Sep 17 00:00:00 2001 From: Ville Syrjälä Date: Mon, 31 Oct 2016 22:37:05 +0200 Subject: drm/i915: Store struct intel_crtc * in {pipe,plane}_to_crtc_mapping[] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lot of users of the {pipe,plane}_to_crtc_mapping[] will end up casting the result to intel_crtc, so let's just store the intel_crtc pointer in the first place. Signed-off-by: Ville Syrjälä Link: http://patchwork.freedesktop.org/patch/msgid/1477946245-14134-7-git-send-email-ville.syrjala@linux.intel.com Reviewed-by: Chris Wilson --- drivers/gpu/drm/i915/i915_debugfs.c | 9 ++-- drivers/gpu/drm/i915/i915_drv.h | 4 +- drivers/gpu/drm/i915/i915_irq.c | 12 +++--- drivers/gpu/drm/i915/intel_display.c | 68 ++++++++++++++---------------- drivers/gpu/drm/i915/intel_drv.h | 7 ++- drivers/gpu/drm/i915/intel_dvo.c | 4 +- drivers/gpu/drm/i915/intel_fifo_underrun.c | 22 +++++----- drivers/gpu/drm/i915/intel_lvds.c | 4 +- drivers/gpu/drm/i915/intel_pm.c | 10 ++--- 9 files changed, 64 insertions(+), 76 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index bf19192dcc3b..1696f7b7321c 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -4065,8 +4065,7 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv, bool enable) { struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[PIPE_A]; struct intel_crtc_state *pipe_config; struct drm_atomic_state *state; int ret = 0; @@ -4134,8 +4133,7 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, { struct drm_device *dev = &dev_priv->drm; struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; - struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev, pipe); enum intel_display_power_domain power_domain; u32 val = 0; /* shut up gcc */ int ret; @@ -4206,8 +4204,7 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, /* real source -> none transition */ if (source == INTEL_PIPE_CRC_SOURCE_NONE) { struct intel_pipe_crc_entry *entries; - struct intel_crtc *crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; DRM_DEBUG_DRIVER("stopping CRCs for pipe %c\n", pipe_name(pipe)); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 98f59863fe6b..eeed752da9df 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -1935,8 +1935,8 @@ struct drm_i915_private { /* Kernel Modesetting */ - struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; - struct drm_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; wait_queue_head_t pending_flip_queue; #ifdef CONFIG_DEBUG_FS diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 4dda2b1eefdb..696dc6f339ac 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -725,8 +725,7 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t high_frame, low_frame; u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; - struct intel_crtc *intel_crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + struct intel_crtc *intel_crtc = dev_priv->pipe_to_crtc_mapping[pipe]; const struct drm_display_mode *mode = &intel_crtc->base.hwmode; htotal = mode->crtc_htotal; @@ -831,8 +830,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, const struct drm_display_mode *mode) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *intel_crtc = dev_priv->pipe_to_crtc_mapping[pipe]; int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; bool in_vbl = true; @@ -967,7 +965,7 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, struct timeval *vblank_time, unsigned flags) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; if (pipe >= INTEL_INFO(dev)->num_pipes) { DRM_ERROR("Invalid crtc %u\n", pipe); @@ -981,7 +979,7 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, return -EINVAL; } - if (!crtc->hwmode.crtc_clock) { + if (!crtc->base.hwmode.crtc_clock) { DRM_DEBUG_KMS("crtc %u is disabled\n", pipe); return -EBUSY; } @@ -989,7 +987,7 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, /* Helper routine in DRM core does all the work: */ return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, vblank_time, flags, - &crtc->hwmode); + &crtc->base.hwmode); } static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 936870d18851..bed201a65f09 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -1030,10 +1030,9 @@ bool intel_crtc_active(struct intel_crtc *crtc) enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - return intel_crtc->config->cpu_transcoder; + return crtc->config->cpu_transcoder; } static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) @@ -1786,8 +1785,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *intel_crtc = dev_priv->pipe_to_crtc_mapping[pipe]; i915_reg_t reg; uint32_t val, pipeconf_val; @@ -7073,7 +7071,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, if (pipe_config->fdi_lanes <= 2) return 0; - other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C)); + other_crtc = intel_get_crtc_for_pipe(dev, PIPE_C); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -7092,7 +7090,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, return -EINVAL; } - other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B)); + other_crtc = intel_get_crtc_for_pipe(dev, PIPE_B); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -8107,8 +8105,7 @@ static void chv_prepare_pll(struct intel_crtc *crtc, int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, const struct dpll *dpll) { - struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev, pipe); struct intel_crtc_state *pipe_config; pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL); @@ -11663,8 +11660,7 @@ static bool pageflip_finished(struct intel_crtc *crtc, void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_flip_work *work; unsigned long flags; @@ -11677,12 +11673,12 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) * lost pageflips) so needs the full irqsave spinlocks. */ spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && !is_mmio_work(work) && - pageflip_finished(intel_crtc, work)) - page_flip_completed(intel_crtc); + pageflip_finished(crtc, work)) + page_flip_completed(crtc); spin_unlock_irqrestore(&dev->event_lock, flags); } @@ -11690,8 +11686,7 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_flip_work *work; unsigned long flags; @@ -11704,12 +11699,12 @@ void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) * lost pageflips) so needs the full irqsave spinlocks. */ spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && is_mmio_work(work) && - pageflip_finished(intel_crtc, work)) - page_flip_completed(intel_crtc); + pageflip_finished(crtc, work)) + page_flip_completed(crtc); spin_unlock_irqrestore(&dev->event_lock, flags); } @@ -12121,8 +12116,7 @@ static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv, void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_flip_work *work; WARN_ON(!in_interrupt()); @@ -12131,19 +12125,19 @@ void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) return; spin_lock(&dev->event_lock); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && !is_mmio_work(work) && - __pageflip_stall_check_cs(dev_priv, intel_crtc, work)) { + __pageflip_stall_check_cs(dev_priv, crtc, work)) { WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n", - work->flip_queued_vblank, intel_crtc_get_vblank_counter(intel_crtc)); - page_flip_completed(intel_crtc); + work->flip_queued_vblank, intel_crtc_get_vblank_counter(crtc)); + page_flip_completed(crtc); work = NULL; } if (work != NULL && !is_mmio_work(work) && - intel_crtc_get_vblank_counter(intel_crtc) - work->flip_queued_vblank > 1) + intel_crtc_get_vblank_counter(crtc) - work->flip_queued_vblank > 1) intel_queue_rps_boost_for_request(work->flip_queued_req); spin_unlock(&dev->event_lock); } @@ -14181,22 +14175,22 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, return; for_each_pipe(dev_priv, pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; if (!((1 << pipe) & crtc_mask)) continue; - ret = drm_crtc_vblank_get(crtc); + ret = drm_crtc_vblank_get(&crtc->base); if (WARN_ON(ret != 0)) { crtc_mask &= ~(1 << pipe); continue; } - last_vblank_count[pipe] = drm_crtc_vblank_count(crtc); + last_vblank_count[pipe] = drm_crtc_vblank_count(&crtc->base); } for_each_pipe(dev_priv, pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; long lret; if (!((1 << pipe) & crtc_mask)) @@ -14204,12 +14198,12 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, lret = wait_event_timeout(dev->vblank[pipe].queue, last_vblank_count[pipe] != - drm_crtc_vblank_count(crtc), + drm_crtc_vblank_count(&crtc->base), msecs_to_jiffies(50)); WARN(!lret, "pipe %c vblank wait timed out\n", pipe_name(pipe)); - drm_crtc_vblank_put(crtc); + drm_crtc_vblank_put(&crtc->base); } } @@ -15317,8 +15311,8 @@ static int intel_crtc_init(struct drm_device *dev, enum pipe pipe) BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL); - dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base; - dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base; + dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = intel_crtc; + dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = intel_crtc; drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs); @@ -16871,7 +16865,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) pipe = 0; if (encoder->get_hw_state(encoder, &pipe)) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + encoder->base.crtc = &crtc->base; crtc->config->output_types |= 1 << encoder->type; encoder->get_config(encoder, crtc->config); @@ -16972,7 +16967,8 @@ intel_modeset_setup_hw_state(struct drm_device *dev) } for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + intel_sanitize_crtc(crtc); intel_dump_pipe_config(crtc, crtc->config, "[setup_hw_state]"); diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 8ae13a6d0cc3..58e33c3a1bdf 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -1028,14 +1028,14 @@ vlv_pipe_to_channel(enum pipe pipe) } } -static inline struct drm_crtc * +static inline struct intel_crtc * intel_get_crtc_for_pipe(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->pipe_to_crtc_mapping[pipe]; } -static inline struct drm_crtc * +static inline struct intel_crtc * intel_get_crtc_for_plane(struct drm_device *dev, int plane) { struct drm_i915_private *dev_priv = to_i915(dev); @@ -1251,8 +1251,7 @@ intel_wait_for_vblank(struct drm_device *dev, int pipe) static inline void intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe) { - const struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + const struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev, pipe); if (crtc->active) intel_wait_for_vblank(dev, pipe); diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index cd574900cd8d..61d78fc36b6d 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -393,12 +393,12 @@ intel_dvo_get_current_mode(struct drm_connector *connector) * its timings to get how the BIOS set up the panel. */ if (dvo_val & DVO_ENABLE) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; int pipe = (dvo_val & DVO_PIPE_B_SELECT) ? 1 : 0; crtc = intel_get_crtc_for_pipe(dev, pipe); if (crtc) { - mode = intel_crtc_mode_get(dev, crtc); + mode = intel_crtc_mode_get(dev, &crtc->base); if (mode) { mode->type |= DRM_MODE_TYPE_PREFERRED; if (dvo_val & DVO_HSYNC_ACTIVE_HIGH) diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c index 3018f4f589c8..34f93f112107 100644 --- a/drivers/gpu/drm/i915/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c @@ -57,7 +57,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = dev_priv->pipe_to_crtc_mapping[pipe]; if (crtc->cpu_fifo_underrun_disabled) return false; @@ -75,7 +75,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = dev_priv->pipe_to_crtc_mapping[pipe]; if (crtc->pch_fifo_underrun_disabled) return false; @@ -245,14 +245,13 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; bool old; assert_spin_locked(&dev_priv->irq_lock); - old = !intel_crtc->cpu_fifo_underrun_disabled; - intel_crtc->cpu_fifo_underrun_disabled = !enable; + old = !crtc->cpu_fifo_underrun_disabled; + crtc->cpu_fifo_underrun_disabled = !enable; if (HAS_GMCH_DISPLAY(dev_priv)) i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old); @@ -314,8 +313,7 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, enum transcoder pch_transcoder, bool enable) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder]; unsigned long flags; bool old; @@ -330,8 +328,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, spin_lock_irqsave(&dev_priv->irq_lock, flags); - old = !intel_crtc->pch_fifo_underrun_disabled; - intel_crtc->pch_fifo_underrun_disabled = !enable; + old = !crtc->pch_fifo_underrun_disabled; + crtc->pch_fifo_underrun_disabled = !enable; if (HAS_PCH_IBX(dev_priv)) ibx_set_fifo_underrun_reporting(&dev_priv->drm, @@ -358,7 +356,7 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; /* We may be called too early in init, thanks BIOS! */ if (crtc == NULL) @@ -366,7 +364,7 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, /* GMCH can't disable fifo underruns, filter them. */ if (HAS_GMCH_DISPLAY(dev_priv) && - to_intel_crtc(crtc)->cpu_fifo_underrun_disabled) + crtc->cpu_fifo_underrun_disabled) return; if (intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false)) diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 199b90c7907a..81ed32b4f899 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -985,7 +985,7 @@ void intel_lvds_init(struct drm_device *dev) struct drm_display_mode *fixed_mode = NULL; struct drm_display_mode *downclock_mode = NULL; struct edid *edid; - struct drm_crtc *crtc; + struct intel_crtc *crtc; i915_reg_t lvds_reg; u32 lvds; int pipe; @@ -1166,7 +1166,7 @@ void intel_lvds_init(struct drm_device *dev) crtc = intel_get_crtc_for_pipe(dev, pipe); if (crtc && (lvds & LVDS_PORT_EN)) { - fixed_mode = intel_crtc_mode_get(dev, crtc); + fixed_mode = intel_crtc_mode_get(dev, &crtc->base); if (fixed_mode) { DRM_DEBUG_KMS("using current (BIOS) mode: "); drm_mode_debug_printmodeline(fixed_mode); diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index f3bd2200d7c5..8d1040c7ed8d 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -728,7 +728,7 @@ static bool g4x_compute_wm0(struct drm_device *dev, int line_time_us, line_count; int entries, tlb_miss; - crtc = to_intel_crtc(intel_get_crtc_for_plane(dev, plane)); + crtc = intel_get_crtc_for_plane(dev, plane); if (!intel_crtc_active(crtc)) { *cursor_wm = cursor->guard_size; *plane_wm = display->guard_size; @@ -823,7 +823,7 @@ static bool g4x_compute_srwm(struct drm_device *dev, return false; } - crtc = to_intel_crtc(intel_get_crtc_for_plane(dev, plane)); + crtc = intel_get_crtc_for_plane(dev, plane); adjusted_mode = &crtc->config->base.adjusted_mode; fb = crtc->base.primary->state->fb; clock = adjusted_mode->crtc_clock; @@ -1546,7 +1546,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) wm_info = &i830_a_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 0); - crtc = to_intel_crtc(intel_get_crtc_for_plane(dev, 0)); + crtc = intel_get_crtc_for_plane(dev, 0); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; @@ -1573,7 +1573,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) wm_info = &i830_bc_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 1); - crtc = to_intel_crtc(intel_get_crtc_for_plane(dev, 1)); + crtc = intel_get_crtc_for_plane(dev, 1); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; @@ -3066,7 +3066,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) /* Since we're now guaranteed to only have one active CRTC... */ pipe = ffs(intel_state->active_crtcs) - 1; - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = dev_priv->pipe_to_crtc_mapping[pipe]; cstate = to_intel_crtc_state(crtc->base.state); if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) -- cgit v1.2.3 From 0f0f74bc83aa2902914d6c4ecee3d5d37338f591 Mon Sep 17 00:00:00 2001 From: Ville Syrjälä Date: Mon, 31 Oct 2016 22:37:06 +0200 Subject: drm/i915: Pass dev_priv to intel_wait_for_vblank() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unify our approach to things by passing around dev_priv instead of dev. Signed-off-by: Ville Syrjälä Link: http://patchwork.freedesktop.org/patch/msgid/1477946245-14134-8-git-send-email-ville.syrjala@linux.intel.com Reviewed-by: Chris Wilson --- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/intel_crt.c | 2 +- drivers/gpu/drm/i915/intel_display.c | 30 ++++++++++++++++-------------- drivers/gpu/drm/i915/intel_dp.c | 18 +++++++++--------- drivers/gpu/drm/i915/intel_drv.h | 10 +++++----- drivers/gpu/drm/i915/intel_hdmi.c | 2 +- drivers/gpu/drm/i915/intel_sdvo.c | 4 ++-- drivers/gpu/drm/i915/intel_tv.c | 6 +++--- 8 files changed, 38 insertions(+), 36 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 1696f7b7321c..ce6aa7eef188 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -4211,7 +4211,7 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, drm_modeset_lock(&crtc->base.mutex, NULL); if (crtc->base.state->active) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); drm_modeset_unlock(&crtc->base.mutex); spin_lock_irq(&pipe_crc->lock); diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index a97151fcb9f4..30eb95b54dcf 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -573,7 +573,7 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe) POSTING_READ(pipeconf_reg); /* Wait for next Vblank to substitue * border color for Color info */ - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); st00 = I915_READ8(_VGA_MSR_WRITE); status = ((st00 & (1 << 4)) != 0) ? connector_status_connected : diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index bed201a65f09..50901b030f80 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -4249,6 +4249,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc) bool intel_has_pending_fb_unpin(struct drm_device *dev) { + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc; /* Note that we don't need to be called with mode_config.lock here @@ -4263,7 +4264,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) continue; if (crtc->flip_work) - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); return true; } @@ -4940,7 +4941,7 @@ void hsw_disable_ips(struct intel_crtc *crtc) } /* We need to wait for a vblank before we can disable the plane. */ - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc) @@ -5052,7 +5053,7 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) if (HAS_GMCH_DISPLAY(dev_priv)) { intel_set_memory_cxsr(dev_priv, false); dev_priv->wm.vlv.cxsr = false; - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); } } @@ -5129,7 +5130,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) if (old_crtc_state->base.active) { intel_set_memory_cxsr(dev_priv, false); dev_priv->wm.vlv.cxsr = false; - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } } @@ -5142,7 +5143,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) */ if (pipe_config->disable_lp_wm) { ilk_disable_lp_wm(dev); - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } /* @@ -5397,7 +5398,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, /* Must wait for vblank to avoid spurious PCH FIFO underruns */ if (intel_crtc->config->has_pch_encoder) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); } @@ -5507,8 +5508,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, intel_encoders_enable(crtc, pipe_config, old_state); if (intel_crtc->config->has_pch_encoder) { - intel_wait_for_vblank(dev, pipe); - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, true); @@ -5518,8 +5519,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, * to change the workaround. */ hsw_workaround_pipe = pipe_config->hsw_workaround_pipe; if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) { - intel_wait_for_vblank(dev, hsw_workaround_pipe); - intel_wait_for_vblank(dev, hsw_workaround_pipe); + intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); + intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); } } @@ -6833,7 +6834,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state, * wait for planes to fully turn off before disabling the pipe. */ if (IS_GEN2(dev_priv)) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_encoders_disable(crtc, old_crtc_state, old_state); @@ -11129,6 +11130,7 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, struct drm_encoder *encoder = &intel_encoder->base; struct drm_crtc *crtc = NULL; struct drm_device *dev = encoder->dev; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_framebuffer *fb; struct drm_mode_config *config = &dev->mode_config; struct drm_atomic_state *state = NULL, *restore_state = NULL; @@ -11281,7 +11283,7 @@ found: old->restore_state = restore_state; /* let the connector get through one full cycle before testing */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); return true; fail: @@ -14277,7 +14279,7 @@ static void intel_update_crtcs(struct drm_atomic_state *state, static void skl_update_crtcs(struct drm_atomic_state *state, unsigned int *crtc_vblank_mask) { - struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = to_i915(state->dev); struct intel_atomic_state *intel_state = to_intel_atomic_state(state); struct drm_crtc *crtc; struct intel_crtc *intel_crtc; @@ -14328,7 +14330,7 @@ static void skl_update_crtcs(struct drm_atomic_state *state, crtc_vblank_mask); if (vbl_wait) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); progress = true; } diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 1063afe5174d..9c1921a2e239 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -2364,7 +2364,7 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp, * 2. Program DP PLL enable */ if (IS_GEN5(dev_priv)) - intel_wait_for_vblank_if_active(&dev_priv->drm, !crtc->pipe); + intel_wait_for_vblank_if_active(dev_priv, !crtc->pipe); intel_dp->DP |= DP_PLL_ENABLE; @@ -3487,7 +3487,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) I915_WRITE(intel_dp->output_reg, DP); POSTING_READ(intel_dp->output_reg); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -3667,7 +3667,7 @@ intel_dp_configure_mst(struct intel_dp *intel_dp) static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int ret = 0; @@ -3688,7 +3688,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) } do { - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { @@ -3711,7 +3711,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) static int intel_dp_sink_crc_start(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int ret; @@ -3739,14 +3739,14 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp) return -EIO; } - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); return 0; } int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int count, ret; @@ -3757,7 +3757,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) return ret; do { - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { @@ -3990,7 +3990,7 @@ intel_dp_retrain_link(struct intel_dp *intel_dp) intel_dp_stop_link_train(intel_dp); /* Keep underrun reporting disabled until things are stable */ - intel_wait_for_vblank(&dev_priv->drm, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); if (crtc->config->has_pch_encoder) diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 58e33c3a1bdf..6c096e4fa7bf 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -1244,17 +1244,17 @@ intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state) (1 << INTEL_OUTPUT_EDP)); } static inline void -intel_wait_for_vblank(struct drm_device *dev, int pipe) +intel_wait_for_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) { - drm_wait_one_vblank(dev, pipe); + drm_wait_one_vblank(&dev_priv->drm, pipe); } static inline void -intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe) +intel_wait_for_vblank_if_active(struct drm_i915_private *dev_priv, int pipe) { - const struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev, pipe); + const struct intel_crtc *crtc = intel_get_crtc_for_pipe(&dev_priv->drm, pipe); if (crtc->active) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); } u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc); diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index af8715f31807..35ada4e1c6cf 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -1164,7 +1164,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder, I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index 49fb95d03d74..5d59a48e3249 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -1472,7 +1472,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder, temp &= ~SDVO_ENABLE; intel_sdvo_write_sdvox(intel_sdvo, temp); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -1509,7 +1509,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder, intel_sdvo_write_sdvox(intel_sdvo, temp); for (i = 0; i < 2; i++) - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); success = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2); /* Warn if the device reported failure to sync. diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index 7118fb55f57f..9212f00d5752 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -856,7 +856,7 @@ intel_enable_tv(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(dev); /* Prevents vblank waits from timing out in intel_tv_detect_type() */ - intel_wait_for_vblank(encoder->base.dev, + intel_wait_for_vblank(dev_priv, to_intel_crtc(encoder->base.crtc)->pipe); I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE); @@ -1238,7 +1238,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv, I915_WRITE(TV_DAC, tv_dac); POSTING_READ(TV_DAC); - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); type = -1; tv_dac = I915_READ(TV_DAC); @@ -1268,7 +1268,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv, POSTING_READ(TV_CTL); /* For unknown reasons the hw barfs if we don't do this vblank wait. */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); /* Restore interrupt config */ if (connector->polled & DRM_CONNECTOR_POLL_HPD) { -- cgit v1.2.3 From b91eb5cce65047eced9d13e8e15b46dced0c3c53 Mon Sep 17 00:00:00 2001 From: Ville Syrjälä Date: Mon, 31 Oct 2016 22:37:09 +0200 Subject: drm/i915: Pass dev_priv to intel_get_crtc_for_pipe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unify our approach to things by passing around dev_priv instead of dev. Signed-off-by: Ville Syrjälä Link: http://patchwork.freedesktop.org/patch/msgid/1477946245-14134-11-git-send-email-ville.syrjala@linux.intel.com Reviewed-by: Chris Wilson --- drivers/gpu/drm/i915/i915_debugfs.c | 3 +-- drivers/gpu/drm/i915/i915_irq.c | 5 +++-- drivers/gpu/drm/i915/intel_display.c | 6 +++--- drivers/gpu/drm/i915/intel_drv.h | 8 +++----- drivers/gpu/drm/i915/intel_dvo.c | 2 +- drivers/gpu/drm/i915/intel_lvds.c | 2 +- drivers/gpu/drm/i915/intel_pm.c | 11 +++++------ 7 files changed, 17 insertions(+), 20 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index ce6aa7eef188..008f91d5df76 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -4131,9 +4131,8 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, enum pipe pipe, enum intel_pipe_crc_source source) { - struct drm_device *dev = &dev_priv->drm; struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; - struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev, pipe); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); enum intel_display_power_domain power_domain; u32 val = 0; /* shut up gcc */ int ret; diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 696dc6f339ac..48bb5debb3ef 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -965,15 +965,16 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, struct timeval *vblank_time, unsigned flags) { + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc; - if (pipe >= INTEL_INFO(dev)->num_pipes) { + if (pipe >= INTEL_INFO(dev_priv)->num_pipes) { DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } /* Get drm_crtc to timestamp: */ - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc == NULL) { DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 419c6a2b3acf..3cb41338894c 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -7072,7 +7072,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, if (pipe_config->fdi_lanes <= 2) return 0; - other_crtc = intel_get_crtc_for_pipe(dev, PIPE_C); + other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -7091,7 +7091,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, return -EINVAL; } - other_crtc = intel_get_crtc_for_pipe(dev, PIPE_B); + other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -8106,7 +8106,7 @@ static void chv_prepare_pll(struct intel_crtc *crtc, int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, const struct dpll *dpll) { - struct intel_crtc *crtc = intel_get_crtc_for_pipe(&dev_priv->drm, pipe); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_crtc_state *pipe_config; pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL); diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 7c0af57819f9..0fdd546ac8d8 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -1029,16 +1029,14 @@ vlv_pipe_to_channel(enum pipe pipe) } static inline struct intel_crtc * -intel_get_crtc_for_pipe(struct drm_device *dev, int pipe) +intel_get_crtc_for_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->pipe_to_crtc_mapping[pipe]; } static inline struct intel_crtc * -intel_get_crtc_for_plane(struct drm_device *dev, int plane) +intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum plane plane) { - struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->plane_to_crtc_mapping[plane]; } @@ -1251,7 +1249,7 @@ intel_wait_for_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) static inline void intel_wait_for_vblank_if_active(struct drm_i915_private *dev_priv, int pipe) { - const struct intel_crtc *crtc = intel_get_crtc_for_pipe(&dev_priv->drm, pipe); + const struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->active) intel_wait_for_vblank(dev_priv, pipe); diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index 61d78fc36b6d..708645443046 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -396,7 +396,7 @@ intel_dvo_get_current_mode(struct drm_connector *connector) struct intel_crtc *crtc; int pipe = (dvo_val & DVO_PIPE_B_SELECT) ? 1 : 0; - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc) { mode = intel_crtc_mode_get(dev, &crtc->base); if (mode) { diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 81ed32b4f899..de7b3e6ed477 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -1163,7 +1163,7 @@ void intel_lvds_init(struct drm_device *dev) goto failed; pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0; - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc && (lvds & LVDS_PORT_EN)) { fixed_mode = intel_crtc_mode_get(dev, &crtc->base); diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index c9990a9f9759..976d66aa8662 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -728,7 +728,7 @@ static bool g4x_compute_wm0(struct drm_i915_private *dev_priv, int line_time_us, line_count; int entries, tlb_miss; - crtc = intel_get_crtc_for_plane(&dev_priv->drm, plane); + crtc = intel_get_crtc_for_plane(dev_priv, plane); if (!intel_crtc_active(crtc)) { *cursor_wm = cursor->guard_size; *plane_wm = display->guard_size; @@ -823,7 +823,7 @@ static bool g4x_compute_srwm(struct drm_i915_private *dev_priv, return false; } - crtc = intel_get_crtc_for_plane(&dev_priv->drm, plane); + crtc = intel_get_crtc_for_plane(dev_priv, plane); adjusted_mode = &crtc->config->base.adjusted_mode; fb = crtc->base.primary->state->fb; clock = adjusted_mode->crtc_clock; @@ -1393,9 +1393,8 @@ static void vlv_update_wm(struct intel_crtc *crtc) static void g4x_update_wm(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); static const int sr_latency_ns = 12000; - struct drm_i915_private *dev_priv = to_i915(dev); int planea_wm, planeb_wm, cursora_wm, cursorb_wm; int plane_sr, cursor_sr; unsigned int enabled = 0; @@ -1546,7 +1545,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) wm_info = &i830_a_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 0); - crtc = intel_get_crtc_for_plane(dev, 0); + crtc = intel_get_crtc_for_plane(dev_priv, 0); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; @@ -1573,7 +1572,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) wm_info = &i830_bc_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 1); - crtc = intel_get_crtc_for_plane(dev, 1); + crtc = intel_get_crtc_for_plane(dev_priv, 1); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; -- cgit v1.2.3 From 98187836fc75b7a003db101e19cfa8685da78905 Mon Sep 17 00:00:00 2001 From: Ville Syrjälä Date: Mon, 31 Oct 2016 22:37:10 +0200 Subject: drm/i915: Always use intel_get_crtc_for_pipe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the open coded dev_priv->pipe_to_crtc_mapping[] usage with intel_get_crtc_for_pipe(). Mostly done with coccinelle, with a few manual tweaks @@ expression E1, E2; @@ ( - E1->pipe_to_crtc_mapping[E2] + intel_get_crtc_for_pipe(E1, E2) | - E1->plane_to_crtc_mapping[E2] + intel_get_crtc_for_plane(E1, E2) ) Signed-off-by: Ville Syrjälä Link: http://patchwork.freedesktop.org/patch/msgid/1477946245-14134-12-git-send-email-ville.syrjala@linux.intel.com Reviewed-by: Chris Wilson --- drivers/gpu/drm/i915/i915_debugfs.c | 5 +++-- drivers/gpu/drm/i915/i915_irq.c | 6 ++++-- drivers/gpu/drm/i915/intel_display.c | 21 ++++++++++++--------- drivers/gpu/drm/i915/intel_fifo_underrun.c | 11 ++++++----- drivers/gpu/drm/i915/intel_pm.c | 2 +- 5 files changed, 26 insertions(+), 19 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 008f91d5df76..a13ff4772e28 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -4065,7 +4065,7 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv, bool enable) { struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[PIPE_A]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A); struct intel_crtc_state *pipe_config; struct drm_atomic_state *state; int ret = 0; @@ -4203,7 +4203,8 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, /* real source -> none transition */ if (source == INTEL_PIPE_CRC_SOURCE_NONE) { struct intel_pipe_crc_entry *entries; - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); DRM_DEBUG_DRIVER("stopping CRCs for pipe %c\n", pipe_name(pipe)); diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 48bb5debb3ef..ecd06d301d8b 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -725,7 +725,8 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t high_frame, low_frame; u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; - struct intel_crtc *intel_crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); const struct drm_display_mode *mode = &intel_crtc->base.hwmode; htotal = mode->crtc_htotal; @@ -830,7 +831,8 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, const struct drm_display_mode *mode) { struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *intel_crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; bool in_vbl = true; diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 3cb41338894c..b21dc272bbac 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -1030,7 +1030,7 @@ bool intel_crtc_active(struct intel_crtc *crtc) enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); return crtc->config->cpu_transcoder; } @@ -1785,7 +1785,8 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct intel_crtc *intel_crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); i915_reg_t reg; uint32_t val, pipeconf_val; @@ -11662,7 +11663,7 @@ static bool pageflip_finished(struct intel_crtc *crtc, void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; unsigned long flags; @@ -11688,7 +11689,7 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; unsigned long flags; @@ -12118,7 +12119,7 @@ static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv, void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; WARN_ON(!in_interrupt()); @@ -14177,7 +14178,8 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, return; for_each_pipe(dev_priv, pipe) { - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); if (!((1 << pipe) & crtc_mask)) continue; @@ -14192,7 +14194,8 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, } for_each_pipe(dev_priv, pipe) { - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); long lret; if (!((1 << pipe) & crtc_mask)) @@ -16867,7 +16870,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) pipe = 0; if (encoder->get_hw_state(encoder, &pipe)) { - crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); encoder->base.crtc = &crtc->base; crtc->config->output_types |= 1 << encoder->type; @@ -16969,7 +16972,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev) } for_each_pipe(dev_priv, pipe) { - crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); intel_sanitize_crtc(crtc); intel_dump_pipe_config(crtc, crtc->config, diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c index 34f93f112107..e660d8b4bbc3 100644 --- a/drivers/gpu/drm/i915/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c @@ -57,7 +57,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->cpu_fifo_underrun_disabled) return false; @@ -75,7 +75,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->pch_fifo_underrun_disabled) return false; @@ -245,7 +245,7 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); bool old; assert_spin_locked(&dev_priv->irq_lock); @@ -313,7 +313,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, enum transcoder pch_transcoder, bool enable) { - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder]; + struct intel_crtc *crtc = + intel_get_crtc_for_pipe(dev_priv, (enum pipe) pch_transcoder); unsigned long flags; bool old; @@ -356,7 +357,7 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct intel_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); /* We may be called too early in init, thanks BIOS! */ if (crtc == NULL) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 976d66aa8662..4f51f8656cf6 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -3065,7 +3065,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) /* Since we're now guaranteed to only have one active CRTC... */ pipe = ffs(intel_state->active_crtcs) - 1; - crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); cstate = to_intel_crtc_state(crtc->base.state); if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) -- cgit v1.2.3 From 56cea32382bcad7eea0a1af5c448089c08ec9f1f Mon Sep 17 00:00:00 2001 From: Joonas Lahtinen Date: Wed, 2 Nov 2016 12:16:04 +0200 Subject: drm/i915: Unify global_list into global_link $ sed -i -r 's/\bglobal_list\b/global_link/g' *.c *.h Cc: Chris Wilson Signed-off-by: Joonas Lahtinen Reviewed-by: Chris Wilson Link: http://patchwork.freedesktop.org/patch/msgid/1478081764-8058-1-git-send-email-joonas.lahtinen@linux.intel.com --- drivers/gpu/drm/i915/i915_debugfs.c | 10 +++++----- drivers/gpu/drm/i915/i915_drv.h | 2 +- drivers/gpu/drm/i915/i915_gem.c | 12 ++++++------ drivers/gpu/drm/i915/i915_gem_gtt.c | 4 ++-- drivers/gpu/drm/i915/i915_gem_shrinker.c | 16 ++++++++-------- drivers/gpu/drm/i915/i915_gem_stolen.c | 2 +- 6 files changed, 23 insertions(+), 23 deletions(-) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index a13ff4772e28..bc9c0cdeb3b3 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -217,7 +217,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) return ret; total_obj_size = total_gtt_size = count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (obj->stolen == NULL) continue; @@ -227,7 +227,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) total_gtt_size += i915_gem_obj_total_ggtt_size(obj); count++; } - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { if (obj->stolen == NULL) continue; @@ -390,7 +390,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) size = count = 0; mapped_size = mapped_count = 0; purgeable_size = purgeable_count = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { size += obj->base.size; ++count; @@ -407,7 +407,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) seq_printf(m, "%u unbound objects, %llu bytes\n", count, size); size = count = dpy_size = dpy_count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { size += obj->base.size; ++count; @@ -493,7 +493,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) return ret; total_obj_size = total_gtt_size = count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (show_pin_display_only && !obj->pin_display) continue; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index d8689792e75e..eaa01da892a4 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -2235,7 +2235,7 @@ struct drm_i915_gem_object { /** Stolen memory for this object, instead of being backed by shmem. */ struct drm_mm_node *stolen; - struct list_head global_list; + struct list_head global_link; union { struct rcu_head rcu; struct llist_node freed; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index a97fdfa80663..5839bebba64a 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1446,7 +1446,7 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) i915 = to_i915(obj->base.dev); list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; - list_move_tail(&obj->global_list, list); + list_move_tail(&obj->global_link, list); } /** @@ -2967,7 +2967,7 @@ int i915_vma_unbind(struct i915_vma *vma) /* Since the unbound list is global, only move to that list if * no more VMAs exist. */ if (--obj->bind_count == 0) - list_move_tail(&obj->global_list, + list_move_tail(&obj->global_link, &to_i915(obj->base.dev)->mm.unbound_list); /* And finally now the object is completely decoupled from this vma, @@ -3164,7 +3164,7 @@ search_free: } GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); - list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); + list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); list_move_tail(&vma->vm_link, &vma->vm->inactive_list); obj->bind_count++; @@ -4125,7 +4125,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, { mutex_init(&obj->mm.lock); - INIT_LIST_HEAD(&obj->global_list); + INIT_LIST_HEAD(&obj->global_link); INIT_LIST_HEAD(&obj->userfault_link); INIT_LIST_HEAD(&obj->obj_exec_link); INIT_LIST_HEAD(&obj->vma_list); @@ -4272,7 +4272,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, GEM_BUG_ON(!list_empty(&obj->vma_list)); GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree)); - list_del(&obj->global_list); + list_del(&obj->global_link); } intel_runtime_pm_put(i915); mutex_unlock(&i915->drm.struct_mutex); @@ -4847,7 +4847,7 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv) i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND); for (p = phases; *p; p++) { - list_for_each_entry(obj, *p, global_list) { + list_for_each_entry(obj, *p, global_link) { obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 67606bf8232b..0c1f5b4750b6 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -3296,7 +3296,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) /* clflush objects bound into the GGTT and rebind them. */ list_for_each_entry_safe(obj, on, - &dev_priv->mm.bound_list, global_list) { + &dev_priv->mm.bound_list, global_link) { bool ggtt_bound = false; struct i915_vma *vma; @@ -3376,7 +3376,7 @@ i915_vma_retire(struct i915_gem_active *active, * (unless we are forced to ofc!) */ if (obj->bind_count) - list_move_tail(&obj->global_list, &rq->i915->mm.bound_list); + list_move_tail(&obj->global_link, &rq->i915->mm.bound_list); obj->mm.dirty = true; /* be paranoid */ diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index 87dd27d5146c..a6fc1bdc48af 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -199,10 +199,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, while (count < target && (obj = list_first_entry_or_null(phase->list, typeof(*obj), - global_list))) { - list_move_tail(&obj->global_list, &still_in_list); + global_link))) { + list_move_tail(&obj->global_link, &still_in_list); if (!obj->mm.pages) { - list_del_init(&obj->global_list); + list_del_init(&obj->global_link); continue; } @@ -228,7 +228,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, I915_MM_SHRINKER); if (!obj->mm.pages) { __i915_gem_object_invalidate(obj); - list_del_init(&obj->global_list); + list_del_init(&obj->global_link); count += obj->base.size >> PAGE_SHIFT; } mutex_unlock(&obj->mm.lock); @@ -293,11 +293,11 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) i915_gem_retire_requests(dev_priv); count = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) if (can_release_pages(obj)) count += obj->base.size >> PAGE_SHIFT; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) count += obj->base.size >> PAGE_SHIFT; } @@ -398,7 +398,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) * being pointed to by hardware. */ unbound = bound = unevictable = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { if (!obj->mm.pages) continue; @@ -407,7 +407,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) else unbound += obj->base.size >> PAGE_SHIFT; } - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (!obj->mm.pages) continue; diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index 3725a1d3b254..b1d367dba347 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -723,7 +723,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, vma->flags |= I915_VMA_GLOBAL_BIND; __i915_vma_set_map_and_fenceable(vma); list_move_tail(&vma->vm_link, &ggtt->base.inactive_list); - list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); + list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); obj->bind_count++; return obj; -- cgit v1.2.3 From c8247c067159782a27c4f2c371c9e26827131a78 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 27 Oct 2016 01:03:43 +0100 Subject: drm/i915: Show the execlist queue in debugfs/i915_engine_info When looking at freezes whilst working on execlists, knowing the order of the pending requests in the driver is useful. Signed-off-by: Chris Wilson Link: http://patchwork.freedesktop.org/patch/msgid/20161027000348.4641-2-chris@chris-wilson.co.uk Reviewed-by: Joonas Lahtinen --- drivers/gpu/drm/i915/i915_debugfs.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'drivers/gpu/drm/i915/i915_debugfs.c') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index bc9c0cdeb3b3..c9465fbff2df 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -3255,6 +3255,12 @@ static int i915_engine_info(struct seq_file *m, void *unused) else seq_printf(m, "\t\tELSP[1] idle\n"); rcu_read_unlock(); + + spin_lock_irq(&engine->execlist_lock); + list_for_each_entry(rq, &engine->execlist_queue, execlist_link) { + print_request(m, rq, "\t\tQ "); + } + spin_unlock_irq(&engine->execlist_lock); } else if (INTEL_GEN(dev_priv) > 6) { seq_printf(m, "\tPP_DIR_BASE: 0x%08x\n", I915_READ(RING_PP_DIR_BASE(engine))); -- cgit v1.2.3