diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
111 files changed, 10955 insertions, 7380 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 1c1b19ccb92f..5ddde7349fbd 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -11,6 +11,7 @@ config DRM_I915 select DRM_KMS_HELPER select DRM_PANEL select DRM_MIPI_DSI + select RELAY # i915 depends on ACPI_VIDEO when ACPI is enabled # but for select to work, need to select ACPI_VIDEO's dependencies, ick select BACKLIGHT_LCD_SUPPORT if ACPI @@ -24,25 +25,31 @@ config DRM_I915 including 830M, 845G, 852GM, 855GM, 865G, 915G, 945G, 965G, G35, G41, G43, G45 chipsets and Celeron, Pentium, Core i3, Core i5, Core i7 as well as Atom CPUs with integrated graphics. - If M is selected, the module will be called i915. AGP support - is required for this driver to work. This driver is used by - the Intel driver in X.org 6.8 and XFree86 4.4 and above. It - replaces the older i830 module that supported a subset of the - hardware in older X.org releases. + + This driver is used by the Intel driver in X.org 6.8 and + XFree86 4.4 and above. It replaces the older i830 module that + supported a subset of the hardware in older X.org releases. Note that the older i810/i815 chipsets require the use of the i810 driver instead, and the Atom z5xx series has an entirely different implementation. -config DRM_I915_PRELIMINARY_HW_SUPPORT - bool "Enable preliminary support for prerelease Intel hardware by default" + If "M" is selected, the module will be called i915. + +config DRM_I915_ALPHA_SUPPORT + bool "Enable alpha quality support for new Intel hardware by default" depends on DRM_I915 default n help - Choose this option if you have prerelease Intel hardware and want the - i915 driver to support it by default. You can enable such support at - runtime with the module option i915.preliminary_hw_support=1; this - option changes the default for that module option. + Choose this option if you have new Intel hardware and want to enable + the alpha quality i915 driver support for the hardware in this kernel + version. You can also enable the support at runtime using the module + parameter i915.alpha_support=1; this option changes the default for + that module parameter. + + It is recommended to upgrade to a kernel version with proper support + as soon as it is available. Generally fixes for platforms with alpha + support are not backported to older kernels. If in doubt, say "N". @@ -85,6 +92,7 @@ config DRM_I915_USERPTR config DRM_I915_GVT bool "Enable Intel GVT-g graphics virtualization host support" depends on DRM_I915 + depends on 64BIT default n help Choose this option if you want to enable Intel GVT-g graphics @@ -104,6 +112,15 @@ config DRM_I915_GVT If in doubt, say "N". +config DRM_I915_GVT_KVMGT + tristate "Enable KVM/VFIO support for Intel GVT-g" + depends on DRM_I915_GVT + depends on KVM + default n + help + Choose this option if you want to enable KVMGT support for + Intel GVT-g. + menu "drm/i915 Debugging" depends on DRM_I915 depends on EXPERT diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug index cee87bfd10c4..51ba630a134b 100644 --- a/drivers/gpu/drm/i915/Kconfig.debug +++ b/drivers/gpu/drm/i915/Kconfig.debug @@ -21,6 +21,7 @@ config DRM_I915_DEBUG select PREEMPT_COUNT select X86_MSR # used by igt/pm_rpm select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks) + select DRM_DEBUG_MM if DRM=y default n help Choose this option to turn on extra driver debugging that may affect diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 612340097f4b..3dea46af9fe6 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -33,18 +33,22 @@ i915-y += i915_cmd_parser.o \ i915_gem_dmabuf.o \ i915_gem_evict.o \ i915_gem_execbuffer.o \ - i915_gem_fence.o \ + i915_gem_fence_reg.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \ i915_gem_shrinker.o \ i915_gem_stolen.o \ i915_gem_tiling.o \ + i915_gem_timeline.o \ i915_gem_userptr.o \ i915_trace_points.o \ + i915_vma.o \ intel_breadcrumbs.o \ intel_engine_cs.o \ + intel_hangcheck.o \ intel_lrc.o \ intel_mocs.o \ intel_ringbuffer.o \ diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile index 34ea4776af70..8a46a7f31d53 100644 --- a/drivers/gpu/drm/i915/gvt/Makefile +++ b/drivers/gpu/drm/i915/gvt/Makefile @@ -3,5 +3,8 @@ GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \ interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \ execlist.o scheduler.o sched_policy.o render.o cmd_parser.o -ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) -Wall -i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) +ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) -Wall +i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) + +CFLAGS_kvmgt.o := -Wno-unused-function +obj-$(CONFIG_DRM_I915_GVT_KVMGT) += $(GVT_DIR)/kvmgt.o diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index 4c687740f5f1..db516382a4d4 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -47,11 +47,9 @@ enum { * Returns: * Zero on success, negative error code if failed. */ -int intel_vgpu_emulate_cfg_read(void *__vgpu, unsigned int offset, +int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { - struct intel_vgpu *vgpu = __vgpu; - if (WARN_ON(bytes > 4)) return -EINVAL; @@ -82,9 +80,8 @@ static int map_aperture(struct intel_vgpu *vgpu, bool map) ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn, first_mfn, - vgpu_aperture_sz(vgpu) - >> PAGE_SHIFT, map, - GVT_MAP_APERTURE); + vgpu_aperture_sz(vgpu) >> + PAGE_SHIFT, map); if (ret) return ret; @@ -235,10 +232,9 @@ static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset, * Returns: * Zero on success, negative error code if failed. */ -int intel_vgpu_emulate_cfg_write(void *__vgpu, unsigned int offset, +int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { - struct intel_vgpu *vgpu = __vgpu; int ret; if (WARN_ON(bytes > 4)) diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index aafb57e26288..d26a092c70e8 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -1145,7 +1145,7 @@ static int skl_decode_mi_display_flip(struct parser_exec_state *s, info->event = PRIMARY_B_FLIP_DONE; break; case MI_DISPLAY_FLIP_SKL_PLANE_1_C: - info->pipe = PIPE_B; + info->pipe = PIPE_C; info->event = PRIMARY_C_FLIP_DONE; break; default: @@ -1201,20 +1201,19 @@ static int gen8_update_plane_mmio_from_mi_display_flip( struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; struct intel_vgpu *vgpu = s->vgpu; -#define write_bits(reg, e, s, v) do { \ - vgpu_vreg(vgpu, reg) &= ~GENMASK(e, s); \ - vgpu_vreg(vgpu, reg) |= (v << s); \ -} while (0) - - write_bits(info->surf_reg, 31, 12, info->surf_val); - if (IS_SKYLAKE(dev_priv)) - write_bits(info->stride_reg, 9, 0, info->stride_val); - else - write_bits(info->stride_reg, 15, 6, info->stride_val); - write_bits(info->ctrl_reg, IS_SKYLAKE(dev_priv) ? 12 : 10, - 10, info->tile_val); - -#undef write_bits + set_mask_bits(&vgpu_vreg(vgpu, info->surf_reg), GENMASK(31, 12), + info->surf_val << 12); + if (IS_SKYLAKE(dev_priv)) { + set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(9, 0), + info->stride_val); + set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(12, 10), + info->tile_val << 10); + } else { + set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(15, 6), + info->stride_val << 6); + set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(10, 10), + info->tile_val << 10); + } vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++; intel_vgpu_trigger_virtual_event(vgpu, info->event); @@ -1419,8 +1418,8 @@ static int cmd_handler_mi_op_2e(struct parser_exec_state *s) static int cmd_handler_mi_op_2f(struct parser_exec_state *s) { int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; - int op_size = ((1 << (cmd_val(s, 0) & GENMASK(20, 19) >> 19)) * - sizeof(u32)); + int op_size = (1 << ((cmd_val(s, 0) & GENMASK(20, 19)) >> 19)) * + sizeof(u32); unsigned long gma, gma_high; int ret = 0; @@ -2538,7 +2537,8 @@ static int scan_workload(struct intel_vgpu_workload *workload) s.rb_va = workload->shadow_ring_buffer_va; s.workload = workload; - if (bypass_scan_mask & (1 << workload->ring_id)) + if ((bypass_scan_mask & (1 << workload->ring_id)) || + gma_head == gma_tail) return 0; ret = ip_gma_set(&s, gma_head); diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c index 7e1da1c563ca..bda85dff7b2a 100644 --- a/drivers/gpu/drm/i915/gvt/edid.c +++ b/drivers/gpu/drm/i915/gvt/edid.c @@ -502,8 +502,7 @@ void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu, * ACK of I2C_WRITE * returned byte if it is READ */ - - aux_data_for_write |= (GVT_AUX_I2C_REPLY_ACK & 0xff) << 24; + aux_data_for_write |= GVT_AUX_I2C_REPLY_ACK << 24; vgpu_vreg(vgpu, offset + 4) = aux_data_for_write; } diff --git a/drivers/gpu/drm/i915/gvt/edid.h b/drivers/gpu/drm/i915/gvt/edid.h index de366b1d5196..f6dfc8b795ec 100644 --- a/drivers/gpu/drm/i915/gvt/edid.h +++ b/drivers/gpu/drm/i915/gvt/edid.h @@ -44,7 +44,7 @@ #define GVT_AUX_I2C_READ 0x1 #define GVT_AUX_I2C_STATUS 0x2 #define GVT_AUX_I2C_MOT 0x4 -#define GVT_AUX_I2C_REPLY_ACK (0x0 << 6) +#define GVT_AUX_I2C_REPLY_ACK 0x0 struct intel_vgpu_edid_data { bool data_valid; diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c index c1f6019d8895..f32bb6f6495c 100644 --- a/drivers/gpu/drm/i915/gvt/execlist.c +++ b/drivers/gpu/drm/i915/gvt/execlist.c @@ -838,23 +838,21 @@ int intel_vgpu_init_execlist(struct intel_vgpu *vgpu) } void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu, - unsigned long ring_bitmap) + unsigned long engine_mask) { - int bit; - struct list_head *pos, *n; - struct intel_vgpu_workload *workload = NULL; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + struct intel_engine_cs *engine; + struct intel_vgpu_workload *pos, *n; + unsigned int tmp; - for_each_set_bit(bit, &ring_bitmap, sizeof(ring_bitmap) * 8) { - if (bit >= I915_NUM_ENGINES) - break; + for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { /* free the unsubmited workload in the queue */ - list_for_each_safe(pos, n, &vgpu->workload_q_head[bit]) { - workload = container_of(pos, - struct intel_vgpu_workload, list); - list_del_init(&workload->list); - free_workload(workload); + list_for_each_entry_safe(pos, n, + &vgpu->workload_q_head[engine->id], list) { + list_del_init(&pos->list); + free_workload(pos); } - init_vgpu_execlist(vgpu, bit); + init_vgpu_execlist(vgpu, engine->id); } } diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h index 635f31c6dcc1..7eced40a1e30 100644 --- a/drivers/gpu/drm/i915/gvt/execlist.h +++ b/drivers/gpu/drm/i915/gvt/execlist.h @@ -183,6 +183,6 @@ int intel_vgpu_init_execlist(struct intel_vgpu *vgpu); int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id); void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu, - unsigned long ring_bitmap); + unsigned long engine_mask); #endif /*_GVT_EXECLIST_H_*/ diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 2cc761328569..7eaaf1c9ed2b 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -138,36 +138,6 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index, memcpy(&(e)->val64, &v, sizeof(v)); \ } while (0) -enum { - GTT_TYPE_INVALID = -1, - - GTT_TYPE_GGTT_PTE, - - GTT_TYPE_PPGTT_PTE_4K_ENTRY, - GTT_TYPE_PPGTT_PTE_2M_ENTRY, - GTT_TYPE_PPGTT_PTE_1G_ENTRY, - - GTT_TYPE_PPGTT_PTE_ENTRY, - - GTT_TYPE_PPGTT_PDE_ENTRY, - GTT_TYPE_PPGTT_PDP_ENTRY, - GTT_TYPE_PPGTT_PML4_ENTRY, - - GTT_TYPE_PPGTT_ROOT_ENTRY, - - GTT_TYPE_PPGTT_ROOT_L3_ENTRY, - GTT_TYPE_PPGTT_ROOT_L4_ENTRY, - - GTT_TYPE_PPGTT_ENTRY, - - GTT_TYPE_PPGTT_PTE_PT, - GTT_TYPE_PPGTT_PDE_PT, - GTT_TYPE_PPGTT_PDP_PT, - GTT_TYPE_PPGTT_PML4_PT, - - GTT_TYPE_MAX, -}; - /* * Mappings between GTT_TYPE* enumerations. * Following information can be found according to the given type: @@ -276,7 +246,7 @@ static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index) pte = readq(addr); #else pte = ioread32(addr); - pte |= ioread32(addr + 4) << 32; + pte |= (u64)ioread32(addr + 4) << 32; #endif return pte; } @@ -842,13 +812,18 @@ static int ppgtt_invalidate_shadow_page_by_shadow_entry(struct intel_vgpu *vgpu, { struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *s; + intel_gvt_gtt_type_t cur_pt_type; if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(e->type)))) return -EINVAL; - if (ops->get_pfn(e) == vgpu->gtt.scratch_page_mfn) - return 0; - + if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY + && e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { + cur_pt_type = get_next_pt_type(e->type) + 1; + if (ops->get_pfn(e) == + vgpu->gtt.scratch_pt[cur_pt_type].page_mfn) + return 0; + } s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e)); if (!s) { gvt_err("vgpu%d: fail to find shadow page: mfn: 0x%lx\n", @@ -999,7 +974,7 @@ fail: } static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt, - struct intel_gvt_gtt_entry *we, unsigned long index) + unsigned long index) { struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt); struct intel_vgpu_shadow_page *sp = &spt->shadow_page; @@ -1008,34 +983,35 @@ static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt, struct intel_gvt_gtt_entry e; int ret; - trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, - we->val64, index); - ppgtt_get_shadow_entry(spt, &e, index); + + trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, e.val64, + index); + if (!ops->test_present(&e)) return 0; - if (ops->get_pfn(&e) == vgpu->gtt.scratch_page_mfn) + if (ops->get_pfn(&e) == vgpu->gtt.scratch_pt[sp->type].page_mfn) return 0; - if (gtt_type_is_pt(get_next_pt_type(we->type))) { - struct intel_vgpu_guest_page *g = - intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we)); - if (!g) { + if (gtt_type_is_pt(get_next_pt_type(e.type))) { + struct intel_vgpu_ppgtt_spt *s = + ppgtt_find_shadow_page(vgpu, ops->get_pfn(&e)); + if (!s) { gvt_err("fail to find guest page\n"); ret = -ENXIO; goto fail; } - ret = ppgtt_invalidate_shadow_page(guest_page_to_ppgtt_spt(g)); + ret = ppgtt_invalidate_shadow_page(s); if (ret) goto fail; } - ops->set_pfn(&e, vgpu->gtt.scratch_page_mfn); + ops->set_pfn(&e, vgpu->gtt.scratch_pt[sp->type].page_mfn); ppgtt_set_shadow_entry(spt, &e, index); return 0; fail: gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n", - vgpu->id, spt, we->val64, we->type); + vgpu->id, spt, e.val64, e.type); return ret; } @@ -1256,23 +1232,16 @@ static int ppgtt_handle_guest_write_page_table( struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt); struct intel_vgpu *vgpu = spt->vgpu; struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; - struct intel_gvt_gtt_entry ge; - int old_present, new_present; int ret; + int new_present; - ppgtt_get_guest_entry(spt, &ge, index); - - old_present = ops->test_present(&ge); new_present = ops->test_present(we); - ppgtt_set_guest_entry(spt, we, index); + ret = ppgtt_handle_guest_entry_removal(gpt, index); + if (ret) + goto fail; - if (old_present) { - ret = ppgtt_handle_guest_entry_removal(gpt, &ge, index); - if (ret) - goto fail; - } if (new_present) { ret = ppgtt_handle_guest_entry_add(gpt, we, index); if (ret) @@ -1318,7 +1287,7 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu) { struct list_head *pos, *n; struct intel_vgpu_ppgtt_spt *spt; - struct intel_gvt_gtt_entry ge, e; + struct intel_gvt_gtt_entry ge; unsigned long index; int ret; @@ -1329,9 +1298,6 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu) for_each_set_bit(index, spt->post_shadow_bitmap, GTT_ENTRY_NUM_IN_ONE_PAGE) { ppgtt_get_guest_entry(spt, &ge, index); - e = ge; - e.val64 = 0; - ppgtt_set_guest_entry(spt, &e, index); ret = ppgtt_handle_guest_write_page_table( &spt->guest_page, &ge, index); @@ -1359,8 +1325,6 @@ static int ppgtt_handle_guest_write_page_table_bytes(void *gp, index = (pa & (PAGE_SIZE - 1)) >> info->gtt_entry_size_shift; ppgtt_get_guest_entry(spt, &we, index); - memcpy((void *)&we.val64 + (pa & (info->gtt_entry_size - 1)), - p_data, bytes); ops->test_pse(&we); @@ -1369,19 +1333,13 @@ static int ppgtt_handle_guest_write_page_table_bytes(void *gp, if (ret) return ret; } else { - struct intel_gvt_gtt_entry ge; - - ppgtt_get_guest_entry(spt, &ge, index); - if (!test_bit(index, spt->post_shadow_bitmap)) { - ret = ppgtt_handle_guest_entry_removal(gpt, - &ge, index); + ret = ppgtt_handle_guest_entry_removal(gpt, index); if (ret) return ret; } ppgtt_set_post_shadow(spt, index); - ppgtt_set_guest_entry(spt, &we, index); } if (!enable_out_of_sync) @@ -1921,47 +1879,101 @@ int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, return ret; } -static int create_scratch_page(struct intel_vgpu *vgpu) +static int alloc_scratch_pages(struct intel_vgpu *vgpu, + intel_gvt_gtt_type_t type) { struct intel_vgpu_gtt *gtt = &vgpu->gtt; - void *p; - void *vaddr; + struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + int page_entry_num = GTT_PAGE_SIZE >> + vgpu->gvt->device_info.gtt_entry_size_shift; + struct page *scratch_pt; unsigned long mfn; + int i; + void *p; + + if (WARN_ON(type < GTT_TYPE_PPGTT_PTE_PT || type >= GTT_TYPE_MAX)) + return -EINVAL; - gtt->scratch_page = alloc_page(GFP_KERNEL); - if (!gtt->scratch_page) { - gvt_err("Failed to allocate scratch page.\n"); + scratch_pt = alloc_page(GFP_KERNEL | GFP_ATOMIC | __GFP_ZERO); + if (!scratch_pt) { + gvt_err("fail to allocate scratch page\n"); return -ENOMEM; } - /* set to zero */ - p = kmap_atomic(gtt->scratch_page); - memset(p, 0, PAGE_SIZE); + p = kmap_atomic(scratch_pt); + mfn = intel_gvt_hypervisor_virt_to_mfn(p); + if (mfn == INTEL_GVT_INVALID_ADDR) { + gvt_err("fail to translate vaddr:0x%llx\n", (u64)p); + kunmap_atomic(p); + __free_page(scratch_pt); + return -EFAULT; + } + gtt->scratch_pt[type].page_mfn = mfn; + gtt->scratch_pt[type].page = scratch_pt; + gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n", + vgpu->id, type, mfn); + + /* Build the tree by full filled the scratch pt with the entries which + * point to the next level scratch pt or scratch page. The + * scratch_pt[type] indicate the scratch pt/scratch page used by the + * 'type' pt. + * e.g. scratch_pt[GTT_TYPE_PPGTT_PDE_PT] is used by + * GTT_TYPE_PPGTT_PDE_PT level pt, that means this scatch_pt it self + * is GTT_TYPE_PPGTT_PTE_PT, and full filled by scratch page mfn. + */ + if (type > GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX) { + struct intel_gvt_gtt_entry se; + + memset(&se, 0, sizeof(struct intel_gvt_gtt_entry)); + se.type = get_entry_type(type - 1); + ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn); + + /* The entry parameters like present/writeable/cache type + * set to the same as i915's scratch page tree. + */ + se.val64 |= _PAGE_PRESENT | _PAGE_RW; + if (type == GTT_TYPE_PPGTT_PDE_PT) + se.val64 |= PPAT_CACHED_INDEX; + + for (i = 0; i < page_entry_num; i++) + ops->set_entry(p, &se, i, false, 0, vgpu); + } + kunmap_atomic(p); - /* translate page to mfn */ - vaddr = page_address(gtt->scratch_page); - mfn = intel_gvt_hypervisor_virt_to_mfn(vaddr); + return 0; +} + +static int release_scratch_page_tree(struct intel_vgpu *vgpu) +{ + int i; - if (mfn == INTEL_GVT_INVALID_ADDR) { - gvt_err("fail to translate vaddr:0x%llx\n", (u64)vaddr); - __free_page(gtt->scratch_page); - gtt->scratch_page = NULL; - return -ENXIO; + for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) { + if (vgpu->gtt.scratch_pt[i].page != NULL) { + __free_page(vgpu->gtt.scratch_pt[i].page); + vgpu->gtt.scratch_pt[i].page = NULL; + vgpu->gtt.scratch_pt[i].page_mfn = 0; + } } - gtt->scratch_page_mfn = mfn; - gvt_dbg_core("vgpu%d create scratch page: mfn=0x%lx\n", vgpu->id, mfn); return 0; } -static void release_scratch_page(struct intel_vgpu *vgpu) +static int create_scratch_page_tree(struct intel_vgpu *vgpu) { - if (vgpu->gtt.scratch_page != NULL) { - __free_page(vgpu->gtt.scratch_page); - vgpu->gtt.scratch_page = NULL; - vgpu->gtt.scratch_page_mfn = 0; + int i, ret; + + for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) { + ret = alloc_scratch_pages(vgpu, i); + if (ret) + goto err; } + + return 0; + +err: + release_scratch_page_tree(vgpu); + return ret; } /** @@ -1995,7 +2007,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu) gtt->ggtt_mm = ggtt_mm; - return create_scratch_page(vgpu); + return create_scratch_page_tree(vgpu); } /** @@ -2014,7 +2026,7 @@ void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu) struct intel_vgpu_mm *mm; ppgtt_free_all_shadow_page(vgpu); - release_scratch_page(vgpu); + release_scratch_page_tree(vgpu); list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) { mm = container_of(pos, struct intel_vgpu_mm, list); diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h index e4dcde78f3f9..d250013bc37b 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.h +++ b/drivers/gpu/drm/i915/gvt/gtt.h @@ -88,6 +88,36 @@ enum { INTEL_GVT_MM_PPGTT, }; +typedef enum { + GTT_TYPE_INVALID = -1, + + GTT_TYPE_GGTT_PTE, + + GTT_TYPE_PPGTT_PTE_4K_ENTRY, + GTT_TYPE_PPGTT_PTE_2M_ENTRY, + GTT_TYPE_PPGTT_PTE_1G_ENTRY, + + GTT_TYPE_PPGTT_PTE_ENTRY, + + GTT_TYPE_PPGTT_PDE_ENTRY, + GTT_TYPE_PPGTT_PDP_ENTRY, + GTT_TYPE_PPGTT_PML4_ENTRY, + + GTT_TYPE_PPGTT_ROOT_ENTRY, + + GTT_TYPE_PPGTT_ROOT_L3_ENTRY, + GTT_TYPE_PPGTT_ROOT_L4_ENTRY, + + GTT_TYPE_PPGTT_ENTRY, + + GTT_TYPE_PPGTT_PTE_PT, + GTT_TYPE_PPGTT_PDE_PT, + GTT_TYPE_PPGTT_PDP_PT, + GTT_TYPE_PPGTT_PML4_PT, + + GTT_TYPE_MAX, +} intel_gvt_gtt_type_t; + struct intel_vgpu_mm { int type; bool initialized; @@ -151,6 +181,12 @@ extern void intel_vgpu_destroy_mm(struct kref *mm_ref); struct intel_vgpu_guest_page; +struct intel_vgpu_scratch_pt { + struct page *page; + unsigned long page_mfn; +}; + + struct intel_vgpu_gtt { struct intel_vgpu_mm *ggtt_mm; unsigned long active_ppgtt_mm_bitmap; @@ -160,8 +196,8 @@ struct intel_vgpu_gtt { atomic_t n_write_protected_guest_page; struct list_head oos_page_list_head; struct list_head post_shadow_list_head; - struct page *scratch_page; - unsigned long scratch_page_mfn; + struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX]; + }; extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu); diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index 31b59d40f3fb..398877c3d2fd 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -44,11 +44,14 @@ static const char * const supported_hypervisors[] = { [INTEL_GVT_HYPERVISOR_KVM] = "KVM", }; -struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops = { +static const struct intel_gvt_ops intel_gvt_ops = { .emulate_cfg_read = intel_vgpu_emulate_cfg_read, .emulate_cfg_write = intel_vgpu_emulate_cfg_write, .emulate_mmio_read = intel_vgpu_emulate_mmio_read, .emulate_mmio_write = intel_vgpu_emulate_mmio_write, + .vgpu_create = intel_gvt_create_vgpu, + .vgpu_destroy = intel_gvt_destroy_vgpu, + .vgpu_reset = intel_gvt_reset_vgpu, }; /** @@ -65,6 +68,8 @@ struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops = { */ int intel_gvt_init_host(void) { + int ret; + if (intel_gvt_host.initialized) return 0; @@ -79,10 +84,12 @@ int intel_gvt_init_host(void) symbol_get(xengt_mpt), "xengt"); intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_XEN; } else { +#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT) /* not in Xen. Try KVMGT */ intel_gvt_host.mpt = try_then_request_module( - symbol_get(kvmgt_mpt), "kvm"); + symbol_get(kvmgt_mpt), "kvmgt"); intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_KVM; +#endif } /* Fail to load MPT modules - bail out */ @@ -90,7 +97,8 @@ int intel_gvt_init_host(void) return -EINVAL; /* Try to detect if we're running in host instead of VM. */ - if (!intel_gvt_hypervisor_detect_host()) + ret = intel_gvt_hypervisor_detect_host(); + if (ret) return -ENODEV; gvt_dbg_core("Running with hypervisor %s in host mode\n", @@ -103,19 +111,20 @@ int intel_gvt_init_host(void) static void init_device_info(struct intel_gvt *gvt) { struct intel_gvt_device_info *info = &gvt->device_info; + struct pci_dev *pdev = gvt->dev_priv->drm.pdev; if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) { info->max_support_vgpus = 8; info->cfg_space_size = 256; info->mmio_size = 2 * 1024 * 1024; info->mmio_bar = 0; - info->msi_cap_offset = IS_SKYLAKE(gvt->dev_priv) ? 0xac : 0x90; info->gtt_start_offset = 8 * 1024 * 1024; info->gtt_entry_size = 8; info->gtt_entry_size_shift = 3; info->gmadr_bytes_in_cmd = 8; info->max_surface_size = 36 * 1024 * 1024; } + info->msi_cap_offset = pdev->msi_cap; } static int gvt_service_thread(void *data) @@ -189,6 +198,9 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv) intel_gvt_clean_mmio_info(gvt); intel_gvt_free_firmware(gvt); + intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt); + intel_gvt_clean_vgpu_types(gvt); + kfree(dev_priv->gvt); dev_priv->gvt = NULL; } @@ -266,10 +278,25 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) if (ret) goto out_clean_cmd_parser; - gvt_dbg_core("gvt device creation is done\n"); + ret = intel_gvt_init_vgpu_types(gvt); + if (ret) + goto out_clean_thread; + + ret = intel_gvt_hypervisor_host_init(&dev_priv->drm.pdev->dev, gvt, + &intel_gvt_ops); + if (ret) { + gvt_err("failed to register gvt-g host device: %d\n", ret); + goto out_clean_types; + } + + gvt_dbg_core("gvt device initialization is done\n"); dev_priv->gvt = gvt; return 0; +out_clean_types: + intel_gvt_clean_vgpu_types(gvt); +out_clean_thread: + clean_service_thread(gvt); out_clean_cmd_parser: intel_gvt_clean_cmd_parser(gvt); out_clean_sched_policy: diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 11df62b542b1..b1a7c8dd4b5f 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -161,6 +161,20 @@ struct intel_vgpu { DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES); struct i915_gem_context *shadow_ctx; struct notifier_block shadow_ctx_notifier_block; + +#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT) + struct { + struct device *mdev; + struct vfio_region *region; + int num_regions; + struct eventfd_ctx *intx_trigger; + struct eventfd_ctx *msi_trigger; + struct rb_root cache; + struct mutex cache_lock; + void *vfio_group; + struct notifier_block iommu_notifier; + } vdev; +#endif }; struct intel_gvt_gm { @@ -190,6 +204,16 @@ struct intel_gvt_opregion { u32 opregion_pa; }; +#define NR_MAX_INTEL_VGPU_TYPES 20 +struct intel_vgpu_type { + char name[16]; + unsigned int max_instance; + unsigned int avail_instance; + unsigned int low_gm_size; + unsigned int high_gm_size; + unsigned int fence; +}; + struct intel_gvt { struct mutex lock; struct drm_i915_private *dev_priv; @@ -205,6 +229,8 @@ struct intel_gvt { struct intel_gvt_opregion opregion; struct intel_gvt_workload_scheduler scheduler; DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS); + struct intel_vgpu_type *types; + unsigned int num_types; struct task_struct *service_thread; wait_queue_head_t service_thread_wq; @@ -231,6 +257,14 @@ void intel_gvt_free_firmware(struct intel_gvt *gvt); int intel_gvt_load_firmware(struct intel_gvt *gvt); /* Aperture/GM space definitions for GVT device */ +#define MB_TO_BYTES(mb) ((mb) << 20ULL) +#define BYTES_TO_MB(b) ((b) >> 20ULL) + +#define HOST_LOW_GM_SIZE MB_TO_BYTES(128) +#define HOST_HIGH_GM_SIZE MB_TO_BYTES(384) +#define HOST_FENCE 4 + +/* Aperture/GM space definitions for GVT device */ #define gvt_aperture_sz(gvt) (gvt->dev_priv->ggtt.mappable_end) #define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.mappable_base) @@ -327,14 +361,19 @@ static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu, * leave the bit 3 - bit 0 unchanged. */ *pval = (val & GENMASK(31, 4)) | (*pval & GENMASK(3, 0)); + } else { + *pval = val; } } -struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, - struct intel_vgpu_creation_params * - param); +int intel_gvt_init_vgpu_types(struct intel_gvt *gvt); +void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt); +struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, + struct intel_vgpu_type *type); void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu); +void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu); + /* validating GM functions */ #define vgpu_gmadr_is_aperture(vgpu, gmadr) \ @@ -369,10 +408,10 @@ int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index, int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index, unsigned long *g_index); -int intel_vgpu_emulate_cfg_read(void *__vgpu, unsigned int offset, +int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes); -int intel_vgpu_emulate_cfg_write(void *__vgpu, unsigned int offset, +int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes); void intel_gvt_clean_opregion(struct intel_gvt *gvt); @@ -382,6 +421,24 @@ void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu); int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa); int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci); +int setup_vgpu_mmio(struct intel_vgpu *vgpu); +void populate_pvinfo_page(struct intel_vgpu *vgpu); + +struct intel_gvt_ops { + int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *, + unsigned int); + int (*emulate_cfg_write)(struct intel_vgpu *, unsigned int, void *, + unsigned int); + int (*emulate_mmio_read)(struct intel_vgpu *, u64, void *, + unsigned int); + int (*emulate_mmio_write)(struct intel_vgpu *, u64, void *, + unsigned int); + struct intel_vgpu *(*vgpu_create)(struct intel_gvt *, + struct intel_vgpu_type *); + void (*vgpu_destroy)(struct intel_vgpu *); + void (*vgpu_reset)(struct intel_vgpu *); +}; + #include "mpt.h" diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 3e74fb3d4aa9..522809710312 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -239,7 +239,11 @@ static int handle_device_reset(struct intel_vgpu *vgpu, unsigned int offset, vgpu->resetting = true; intel_vgpu_stop_schedule(vgpu); - if (scheduler->current_vgpu == vgpu) { + /* + * The current_vgpu will set to NULL after stopping the + * scheduler when the reset is triggered by current vgpu. + */ + if (scheduler->current_vgpu == NULL) { mutex_unlock(&vgpu->gvt->lock); intel_gvt_wait_vgpu_idle(vgpu); mutex_lock(&vgpu->gvt->lock); @@ -247,6 +251,16 @@ static int handle_device_reset(struct intel_vgpu *vgpu, unsigned int offset, intel_vgpu_reset_execlist(vgpu, bitmap); + /* full GPU reset */ + if (bitmap == 0xff) { + mutex_unlock(&vgpu->gvt->lock); + intel_vgpu_clean_gtt(vgpu); + mutex_lock(&vgpu->gvt->lock); + setup_vgpu_mmio(vgpu); + populate_pvinfo_page(vgpu); + intel_vgpu_init_gtt(vgpu); + } + vgpu->resetting = false; return 0; @@ -258,6 +272,7 @@ static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, u32 data; u64 bitmap = 0; + write_vreg(vgpu, offset, p_data, bytes); data = vgpu_vreg(vgpu, offset); if (data & GEN6_GRDOM_FULL) { @@ -1143,7 +1158,10 @@ static int fpga_dbg_mmio_write(struct intel_vgpu *vgpu, static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { - u32 mode = *(u32 *)p_data; + u32 mode; + + write_vreg(vgpu, offset, p_data, bytes); + mode = vgpu_vreg(vgpu, offset); if (GFX_MODE_BIT_SET_IN_MASK(mode, START_DMA)) { WARN_ONCE(1, "VM(%d): iGVT-g doesn't supporte GuC\n", @@ -1260,19 +1278,18 @@ static int skl_misc_ctl_write(struct intel_vgpu *vgpu, unsigned int offset, switch (offset) { case 0x4ddc: vgpu_vreg(vgpu, offset) = 0x8000003c; + /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl */ + I915_WRITE(reg, vgpu_vreg(vgpu, offset)); break; case 0x42080: vgpu_vreg(vgpu, offset) = 0x8000; + /* WaCompressedResourceDisplayNewHashMode:skl */ + I915_WRITE(reg, vgpu_vreg(vgpu, offset)); break; default: return -EINVAL; } - /** - * TODO: need detect stepping info after gvt contain such information - * 0x4ddc enabled after C0, 0x42080 enabled after E0. - */ - I915_WRITE(reg, vgpu_vreg(vgpu, offset)); return 0; } @@ -1305,7 +1322,7 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset); struct intel_vgpu_execlist *execlist; u32 data = *(u32 *)p_data; - int ret; + int ret = 0; if (WARN_ON(ring_id < 0 || ring_id > I915_NUM_ENGINES - 1)) return -EINVAL; @@ -1313,12 +1330,15 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, execlist = &vgpu->execlist[ring_id]; execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data; - if (execlist->elsp_dwords.index == 3) + if (execlist->elsp_dwords.index == 3) { ret = intel_vgpu_submit_execlist(vgpu, ring_id); + if(ret) + gvt_err("fail submit workload on ring %d\n", ring_id); + } ++execlist->elsp_dwords.index; execlist->elsp_dwords.index &= 0x3; - return 0; + return ret; } static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, @@ -1349,6 +1369,9 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu, int rc = 0; unsigned int id = 0; + write_vreg(vgpu, offset, p_data, bytes); + vgpu_vreg(vgpu, offset) = 0; + switch (offset) { case 0x4260: id = RCS; @@ -1374,6 +1397,23 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu, return rc; } +static int ring_reset_ctl_write(struct intel_vgpu *vgpu, + unsigned int offset, void *p_data, unsigned int bytes) +{ + u32 data; + + write_vreg(vgpu, offset, p_data, bytes); + data = vgpu_vreg(vgpu, offset); + + if (data & _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)) + data |= RESET_CTL_READY_TO_RESET; + else if (data & _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)) + data &= ~RESET_CTL_READY_TO_RESET; + + vgpu_vreg(vgpu, offset) = data; + return 0; +} + #define MMIO_F(reg, s, f, am, rm, d, r, w) do { \ ret = new_mmio_info(gvt, INTEL_GVT_MMIO_OFFSET(reg), \ f, s, am, rm, d, r, w); \ @@ -1467,7 +1507,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK, NULL, NULL); MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK, NULL, NULL); - MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK, NULL, NULL); + MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK, NULL, NULL); MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK, NULL, NULL); @@ -1476,7 +1516,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_DFH(0x2470, D_ALL, F_MODE_MASK, NULL, NULL); MMIO_D(GAM_ECOCHK, D_ALL); MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK, NULL, NULL); - MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK, NULL, NULL); + MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_D(0x9030, D_ALL); MMIO_D(0x20a0, D_ALL); MMIO_D(0x2420, D_ALL); @@ -1485,7 +1525,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(0x2438, D_ALL); MMIO_D(0x243c, D_ALL); MMIO_DFH(0x7018, D_ALL, F_MODE_MASK, NULL, NULL); - MMIO_DFH(0xe184, D_ALL, F_MODE_MASK, NULL, NULL); + MMIO_DFH(HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(0xe100, D_ALL, F_MODE_MASK, NULL, NULL); /* display */ @@ -2098,6 +2138,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(GEN6_MBCTL, D_ALL); MMIO_D(0x911c, D_ALL); MMIO_D(0x9120, D_ALL); + MMIO_DFH(GEN7_UCGCTL4, D_ALL, F_CMD_ACCESS, NULL, NULL); MMIO_D(GAB_CTL, D_ALL); MMIO_D(0x48800, D_ALL); @@ -2280,6 +2321,15 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_RING_D(RING_ACTHD_UDW, D_BDW_PLUS); +#define RING_REG(base) (base + 0xd0) + MMIO_RING_F(RING_REG, 4, F_RO, 0, + ~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL, + ring_reset_ctl_write); + MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 4, F_RO, 0, + ~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL, + ring_reset_ctl_write); +#undef RING_REG + #define RING_REG(base) (base + 0x230) MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write); MMIO_DH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, elsp_mmio_write); @@ -2327,7 +2377,7 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_RING_GM(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL); MMIO_GM(0x1c080, D_BDW_PLUS, NULL, NULL); - MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); + MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW); MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW); @@ -2346,7 +2396,7 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS); MMIO_D(0xfdc, D_BDW); - MMIO_D(GEN8_ROW_CHICKEN, D_BDW_PLUS); + MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_D(GEN7_ROW_CHICKEN2, D_BDW_PLUS); MMIO_D(GEN8_UCGCTL6, D_BDW_PLUS); @@ -2357,10 +2407,10 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_D(0xb10c, D_BDW); MMIO_D(0xb110, D_BDW); - MMIO_DH(0x24d0, D_BDW_PLUS, NULL, NULL); - MMIO_DH(0x24d4, D_BDW_PLUS, NULL, NULL); - MMIO_DH(0x24d8, D_BDW_PLUS, NULL, NULL); - MMIO_DH(0x24dc, D_BDW_PLUS, NULL, NULL); + MMIO_DFH(0x24d0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(0x24d4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(0x24d8, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(0x24dc, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_D(0x83a4, D_BDW); MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS); @@ -2374,9 +2424,9 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_D(0x6e570, D_BDW_PLUS); MMIO_D(0x65f10, D_BDW_PLUS); - MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); - MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); - MMIO_DFH(0xe180, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); + MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); MMIO_D(0x2248, D_BDW); @@ -2407,6 +2457,7 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_D(0xa210, D_SKL_PLUS); MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS); MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS); + MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DH(0x4ddc, D_SKL, NULL, skl_misc_ctl_write); MMIO_DH(0x42080, D_SKL, NULL, skl_misc_ctl_write); MMIO_D(0x45504, D_SKL); @@ -2556,8 +2607,8 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_D(0x51000, D_SKL); MMIO_D(0x6c00c, D_SKL); - MMIO_F(0xc800, 0x7f8, 0, 0, 0, D_SKL, NULL, NULL); - MMIO_F(0xb020, 0x80, 0, 0, 0, D_SKL, NULL, NULL); + MMIO_F(0xc800, 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL, NULL, NULL); + MMIO_F(0xb020, 0x80, F_CMD_ACCESS, 0, 0, D_SKL, NULL, NULL); MMIO_D(0xd08, D_SKL); MMIO_D(0x20e0, D_SKL); diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h index 027ef558d91c..30e543f5a703 100644 --- a/drivers/gpu/drm/i915/gvt/hypercall.h +++ b/drivers/gpu/drm/i915/gvt/hypercall.h @@ -33,21 +33,14 @@ #ifndef _GVT_HYPERCALL_H_ #define _GVT_HYPERCALL_H_ -struct intel_gvt_io_emulation_ops { - int (*emulate_cfg_read)(void *, unsigned int, void *, unsigned int); - int (*emulate_cfg_write)(void *, unsigned int, void *, unsigned int); - int (*emulate_mmio_read)(void *, u64, void *, unsigned int); - int (*emulate_mmio_write)(void *, u64, void *, unsigned int); -}; - -extern struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops; - /* * Specific GVT-g MPT modules function collections. Currently GVT-g supports * both Xen and KVM by providing dedicated hypervisor-related MPT modules. */ struct intel_gvt_mpt { int (*detect_host)(void); + int (*host_init)(struct device *dev, void *gvt, const void *ops); + void (*host_exit)(struct device *dev, void *gvt); int (*attach_vgpu)(void *vgpu, unsigned long *handle); void (*detach_vgpu)(unsigned long handle); int (*inject_msi)(unsigned long handle, u32 addr, u16 data); @@ -60,8 +53,7 @@ struct intel_gvt_mpt { unsigned long len); unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn); int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn, - unsigned long mfn, unsigned int nr, bool map, - int type); + unsigned long mfn, unsigned int nr, bool map); int (*set_trap_area)(unsigned long handle, u64 start, u64 end, bool map); }; diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c new file mode 100644 index 000000000000..dc0365033157 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -0,0 +1,597 @@ +/* + * KVMGT - the implementation of Intel mediated pass-through framework for KVM + * + * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Kevin Tian <kevin.tian@intel.com> + * Jike Song <jike.song@intel.com> + * Xiaoguang Chen <xiaoguang.chen@intel.com> + */ + +#include <linux/init.h> +#include <linux/device.h> +#include <linux/mm.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/rbtree.h> +#include <linux/spinlock.h> +#include <linux/eventfd.h> +#include <linux/uuid.h> +#include <linux/kvm_host.h> +#include <linux/vfio.h> + +#include "i915_drv.h" +#include "gvt.h" + +static inline long kvmgt_pin_pages(struct device *dev, unsigned long *user_pfn, + long npage, int prot, unsigned long *phys_pfn) +{ + return 0; +} +static inline long kvmgt_unpin_pages(struct device *dev, unsigned long *pfn, + long npage) +{ + return 0; +} + +static const struct intel_gvt_ops *intel_gvt_ops; + + +/* helper macros copied from vfio-pci */ +#define VFIO_PCI_OFFSET_SHIFT 40 +#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) +#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) +#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) + +struct vfio_region { + u32 type; + u32 subtype; + size_t size; + u32 flags; +}; + +struct kvmgt_pgfn { + gfn_t gfn; + struct hlist_node hnode; +}; + +struct kvmgt_guest_info { + struct kvm *kvm; + struct intel_vgpu *vgpu; + struct kvm_page_track_notifier_node track_node; +#define NR_BKT (1 << 18) + struct hlist_head ptable[NR_BKT]; +#undef NR_BKT +}; + +struct gvt_dma { + struct rb_node node; + gfn_t gfn; + kvm_pfn_t pfn; +}; + +static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) +{ + struct rb_node *node = vgpu->vdev.cache.rb_node; + struct gvt_dma *ret = NULL; + + while (node) { + struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); + + if (gfn < itr->gfn) + node = node->rb_left; + else if (gfn > itr->gfn) + node = node->rb_right; + else { + ret = itr; + goto out; + } + } + +out: + return ret; +} + +static kvm_pfn_t gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) +{ + struct gvt_dma *entry; + + mutex_lock(&vgpu->vdev.cache_lock); + entry = __gvt_cache_find(vgpu, gfn); + mutex_unlock(&vgpu->vdev.cache_lock); + + return entry == NULL ? 0 : entry->pfn; +} + +static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, kvm_pfn_t pfn) +{ + struct gvt_dma *new, *itr; + struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; + + new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); + if (!new) + return; + + new->gfn = gfn; + new->pfn = pfn; + + mutex_lock(&vgpu->vdev.cache_lock); + while (*link) { + parent = *link; + itr = rb_entry(parent, struct gvt_dma, node); + + if (gfn == itr->gfn) + goto out; + else if (gfn < itr->gfn) + link = &parent->rb_left; + else + link = &parent->rb_right; + } + + rb_link_node(&new->node, parent, link); + rb_insert_color(&new->node, &vgpu->vdev.cache); + mutex_unlock(&vgpu->vdev.cache_lock); + return; + +out: + mutex_unlock(&vgpu->vdev.cache_lock); + kfree(new); +} + +static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, + struct gvt_dma *entry) +{ + rb_erase(&entry->node, &vgpu->vdev.cache); + kfree(entry); +} + +static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn) +{ + struct device *dev = vgpu->vdev.mdev; + struct gvt_dma *this; + unsigned long pfn; + + mutex_lock(&vgpu->vdev.cache_lock); + this = __gvt_cache_find(vgpu, gfn); + if (!this) { + mutex_unlock(&vgpu->vdev.cache_lock); + return; + } + + pfn = this->pfn; + WARN_ON((kvmgt_unpin_pages(dev, &pfn, 1) != 1)); + __gvt_cache_remove_entry(vgpu, this); + mutex_unlock(&vgpu->vdev.cache_lock); +} + +static void gvt_cache_init(struct intel_vgpu *vgpu) +{ + vgpu->vdev.cache = RB_ROOT; + mutex_init(&vgpu->vdev.cache_lock); +} + +static void gvt_cache_destroy(struct intel_vgpu *vgpu) +{ + struct gvt_dma *dma; + struct rb_node *node = NULL; + struct device *dev = vgpu->vdev.mdev; + unsigned long pfn; + + mutex_lock(&vgpu->vdev.cache_lock); + while ((node = rb_first(&vgpu->vdev.cache))) { + dma = rb_entry(node, struct gvt_dma, node); + pfn = dma->pfn; + + kvmgt_unpin_pages(dev, &pfn, 1); + __gvt_cache_remove_entry(vgpu, dma); + } + mutex_unlock(&vgpu->vdev.cache_lock); +} + +static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt, + const char *name) +{ + int i; + struct intel_vgpu_type *t; + const char *driver_name = dev_driver_string( + &gvt->dev_priv->drm.pdev->dev); + + for (i = 0; i < gvt->num_types; i++) { + t = &gvt->types[i]; + if (!strncmp(t->name, name + strlen(driver_name) + 1, + sizeof(t->name))) + return t; + } + + return NULL; +} + +static struct attribute *type_attrs[] = { + NULL, +}; + +static struct attribute_group *intel_vgpu_type_groups[] = { + [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, +}; + +static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i, j; + struct intel_vgpu_type *type; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + type = &gvt->types[i]; + + group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); + if (WARN_ON(!group)) + goto unwind; + + group->name = type->name; + group->attrs = type_attrs; + intel_vgpu_type_groups[i] = group; + } + + return true; + +unwind: + for (j = 0; j < i; j++) { + group = intel_vgpu_type_groups[j]; + kfree(group); + } + + return false; +} + +static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + group = intel_vgpu_type_groups[i]; + kfree(group); + } +} + +static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) +{ + hash_init(info->ptable); +} + +static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) +{ + struct kvmgt_pgfn *p; + struct hlist_node *tmp; + int i; + + hash_for_each_safe(info->ptable, i, tmp, p, hnode) { + hash_del(&p->hnode); + kfree(p); + } +} + +static struct kvmgt_pgfn * +__kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) +{ + struct kvmgt_pgfn *p, *res = NULL; + + hash_for_each_possible(info->ptable, p, hnode, gfn) { + if (gfn == p->gfn) { + res = p; + break; + } + } + + return res; +} + +static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, + gfn_t gfn) +{ + struct kvmgt_pgfn *p; + + p = __kvmgt_protect_table_find(info, gfn); + return !!p; +} + +static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) +{ + struct kvmgt_pgfn *p; + + if (kvmgt_gfn_is_write_protected(info, gfn)) + return; + + p = kmalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); + if (WARN(!p, "gfn: 0x%llx\n", gfn)) + return; + + p->gfn = gfn; + hash_add(info->ptable, &p->hnode, gfn); +} + +static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, + gfn_t gfn) +{ + struct kvmgt_pgfn *p; + + p = __kvmgt_protect_table_find(info, gfn); + if (p) { + hash_del(&p->hnode); + kfree(p); + } +} + +static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) +{ + if (!intel_gvt_init_vgpu_type_groups(gvt)) + return -EFAULT; + + intel_gvt_ops = ops; + + /* MDEV is not yet available */ + return -ENODEV; +} + +static void kvmgt_host_exit(struct device *dev, void *gvt) +{ + intel_gvt_cleanup_vgpu_type_groups(gvt); +} + +static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) +{ + struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle; + struct kvm *kvm = info->kvm; + struct kvm_memory_slot *slot; + int idx; + + idx = srcu_read_lock(&kvm->srcu); + slot = gfn_to_memslot(kvm, gfn); + + spin_lock(&kvm->mmu_lock); + + if (kvmgt_gfn_is_write_protected(info, gfn)) + goto out; + + kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); + kvmgt_protect_table_add(info, gfn); + +out: + spin_unlock(&kvm->mmu_lock); + srcu_read_unlock(&kvm->srcu, idx); + return 0; +} + +static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) +{ + struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle; + struct kvm *kvm = info->kvm; + struct kvm_memory_slot *slot; + int idx; + + idx = srcu_read_lock(&kvm->srcu); + slot = gfn_to_memslot(kvm, gfn); + + spin_lock(&kvm->mmu_lock); + + if (!kvmgt_gfn_is_write_protected(info, gfn)) + goto out; + + kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); + kvmgt_protect_table_del(info, gfn); + +out: + spin_unlock(&kvm->mmu_lock); + srcu_read_unlock(&kvm->srcu, idx); + return 0; +} + +static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, + const u8 *val, int len, + struct kvm_page_track_notifier_node *node) +{ + struct kvmgt_guest_info *info = container_of(node, + struct kvmgt_guest_info, track_node); + + if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) + intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa, + (void *)val, len); +} + +static void kvmgt_page_track_flush_slot(struct kvm *kvm, + struct kvm_memory_slot *slot, + struct kvm_page_track_notifier_node *node) +{ + int i; + gfn_t gfn; + struct kvmgt_guest_info *info = container_of(node, + struct kvmgt_guest_info, track_node); + + spin_lock(&kvm->mmu_lock); + for (i = 0; i < slot->npages; i++) { + gfn = slot->base_gfn + i; + if (kvmgt_gfn_is_write_protected(info, gfn)) { + kvm_slot_page_track_remove_page(kvm, slot, gfn, + KVM_PAGE_TRACK_WRITE); + kvmgt_protect_table_del(info, gfn); + } + } + spin_unlock(&kvm->mmu_lock); +} + +static bool kvmgt_check_guest(void) +{ + unsigned int eax, ebx, ecx, edx; + char s[12]; + unsigned int *i; + + eax = KVM_CPUID_SIGNATURE; + ebx = ecx = edx = 0; + + asm volatile ("cpuid" + : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) + : + : "cc", "memory"); + i = (unsigned int *)s; + i[0] = ebx; + i[1] = ecx; + i[2] = edx; + + return !strncmp(s, "KVMKVMKVM", strlen("KVMKVMKVM")); +} + +/** + * NOTE: + * It's actually impossible to check if we are running in KVM host, + * since the "KVM host" is simply native. So we only dectect guest here. + */ +static int kvmgt_detect_host(void) +{ +#ifdef CONFIG_INTEL_IOMMU + if (intel_iommu_gfx_mapped) { + gvt_err("Hardware IOMMU compatibility not yet supported, try to boot with intel_iommu=igfx_off\n"); + return -ENODEV; + } +#endif + return kvmgt_check_guest() ? -ENODEV : 0; +} + +static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) +{ + /* nothing to do here */ + return 0; +} + +static void kvmgt_detach_vgpu(unsigned long handle) +{ + /* nothing to do here */ +} + +static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) +{ + struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle; + struct intel_vgpu *vgpu = info->vgpu; + + if (vgpu->vdev.msi_trigger) + return eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1; + + return false; +} + +static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) +{ + unsigned long pfn; + struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle; + int rc; + + pfn = gvt_cache_find(info->vgpu, gfn); + if (pfn != 0) + return pfn; + + rc = kvmgt_pin_pages(info->vgpu->vdev.mdev, &gfn, 1, + IOMMU_READ | IOMMU_WRITE, &pfn); + if (rc != 1) { + gvt_err("vfio_pin_pages failed for gfn: 0x%lx\n", gfn); + return 0; + } + + gvt_cache_add(info->vgpu, gfn, pfn); + return pfn; +} + +static void *kvmgt_gpa_to_hva(unsigned long handle, unsigned long gpa) +{ + unsigned long pfn; + gfn_t gfn = gpa_to_gfn(gpa); + + pfn = kvmgt_gfn_to_pfn(handle, gfn); + if (!pfn) + return NULL; + + return (char *)pfn_to_kaddr(pfn) + offset_in_page(gpa); +} + +static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, + void *buf, unsigned long len, bool write) +{ + void *hva = NULL; + + hva = kvmgt_gpa_to_hva(handle, gpa); + if (!hva) + return -EFAULT; + + if (write) + memcpy(hva, buf, len); + else + memcpy(buf, hva, len); + + return 0; +} + +static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, + void *buf, unsigned long len) +{ + return kvmgt_rw_gpa(handle, gpa, buf, len, false); +} + +static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, + void *buf, unsigned long len) +{ + return kvmgt_rw_gpa(handle, gpa, buf, len, true); +} + +static unsigned long kvmgt_virt_to_pfn(void *addr) +{ + return PFN_DOWN(__pa(addr)); +} + +struct intel_gvt_mpt kvmgt_mpt = { + .detect_host = kvmgt_detect_host, + .host_init = kvmgt_host_init, + .host_exit = kvmgt_host_exit, + .attach_vgpu = kvmgt_attach_vgpu, + .detach_vgpu = kvmgt_detach_vgpu, + .inject_msi = kvmgt_inject_msi, + .from_virt_to_mfn = kvmgt_virt_to_pfn, + .set_wp_page = kvmgt_write_protect_add, + .unset_wp_page = kvmgt_write_protect_remove, + .read_gpa = kvmgt_read_gpa, + .write_gpa = kvmgt_write_gpa, + .gfn_to_mfn = kvmgt_gfn_to_pfn, +}; +EXPORT_SYMBOL_GPL(kvmgt_mpt); + +static int __init kvmgt_init(void) +{ + return 0; +} + +static void __exit kvmgt_exit(void) +{ +} + +module_init(kvmgt_init); +module_exit(kvmgt_exit); + +MODULE_LICENSE("GPL and additional rights"); +MODULE_AUTHOR("Intel Corporation"); diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index 585b01f63254..09c9450a1946 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -67,10 +67,9 @@ int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa) * Returns: * Zero on success, negative error code if failed */ -int intel_vgpu_emulate_mmio_read(void *__vgpu, uint64_t pa, +int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa, void *p_data, unsigned int bytes) { - struct intel_vgpu *vgpu = __vgpu; struct intel_gvt *gvt = vgpu->gvt; struct intel_gvt_mmio_info *mmio; unsigned int offset = 0; @@ -179,10 +178,9 @@ err: * Returns: * Zero on success, negative error code if failed */ -int intel_vgpu_emulate_mmio_write(void *__vgpu, uint64_t pa, +int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa, void *p_data, unsigned int bytes) { - struct intel_vgpu *vgpu = __vgpu; struct intel_gvt *gvt = vgpu->gvt; struct intel_gvt_mmio_info *mmio; unsigned int offset = 0; diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h index 9dc739a01892..87d5b5e366a3 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.h +++ b/drivers/gpu/drm/i915/gvt/mmio.h @@ -87,10 +87,11 @@ struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt, }) int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa); -int intel_vgpu_emulate_mmio_read(void *__vgpu, u64 pa, void *p_data, - unsigned int bytes); -int intel_vgpu_emulate_mmio_write(void *__vgpu, u64 pa, void *p_data, - unsigned int bytes); + +int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa, + void *p_data, unsigned int bytes); +int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, u64 pa, + void *p_data, unsigned int bytes); bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt, unsigned int offset); bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt, unsigned int offset); diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h index 67858782d327..1af5830c0a56 100644 --- a/drivers/gpu/drm/i915/gvt/mpt.h +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -56,6 +56,35 @@ static inline int intel_gvt_hypervisor_detect_host(void) } /** + * intel_gvt_hypervisor_host_init - init GVT-g host side + * + * Returns: + * Zero on success, negative error code if failed + */ +static inline int intel_gvt_hypervisor_host_init(struct device *dev, + void *gvt, const void *ops) +{ + /* optional to provide */ + if (!intel_gvt_host.mpt->host_init) + return 0; + + return intel_gvt_host.mpt->host_init(dev, gvt, ops); +} + +/** + * intel_gvt_hypervisor_host_exit - exit GVT-g host side + */ +static inline void intel_gvt_hypervisor_host_exit(struct device *dev, + void *gvt) +{ + /* optional to provide */ + if (!intel_gvt_host.mpt->host_exit) + return; + + intel_gvt_host.mpt->host_exit(dev, gvt); +} + +/** * intel_gvt_hypervisor_attach_vgpu - call hypervisor to initialize vGPU * related stuffs inside hypervisor. * @@ -64,6 +93,10 @@ static inline int intel_gvt_hypervisor_detect_host(void) */ static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu) { + /* optional to provide */ + if (!intel_gvt_host.mpt->attach_vgpu) + return 0; + return intel_gvt_host.mpt->attach_vgpu(vgpu, &vgpu->handle); } @@ -76,6 +109,10 @@ static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu) */ static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu) { + /* optional to provide */ + if (!intel_gvt_host.mpt->detach_vgpu) + return; + intel_gvt_host.mpt->detach_vgpu(vgpu->handle); } @@ -224,11 +261,6 @@ static inline unsigned long intel_gvt_hypervisor_gfn_to_mfn( return intel_gvt_host.mpt->gfn_to_mfn(vgpu->handle, gfn); } -enum { - GVT_MAP_APERTURE = 0, - GVT_MAP_OPREGION, -}; - /** * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN * @vgpu: a vGPU @@ -236,7 +268,6 @@ enum { * @mfn: host PFN * @nr: amount of PFNs * @map: map or unmap - * @type: map type * * Returns: * Zero on success, negative error code if failed. @@ -244,10 +275,14 @@ enum { static inline int intel_gvt_hypervisor_map_gfn_to_mfn( struct intel_vgpu *vgpu, unsigned long gfn, unsigned long mfn, unsigned int nr, - bool map, int type) + bool map) { + /* a MPT implementation could have MMIO mapped elsewhere */ + if (!intel_gvt_host.mpt->map_gfn_to_mfn) + return 0; + return intel_gvt_host.mpt->map_gfn_to_mfn(vgpu->handle, gfn, mfn, nr, - map, type); + map); } /** @@ -263,6 +298,10 @@ static inline int intel_gvt_hypervisor_map_gfn_to_mfn( static inline int intel_gvt_hypervisor_set_trap_area( struct intel_vgpu *vgpu, u64 start, u64 end, bool map) { + /* a MPT implementation could have MMIO trapped elsewhere */ + if (!intel_gvt_host.mpt->set_trap_area) + return 0; + return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map); } diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c index 973c8a9d0b15..d2a0fbc896c3 100644 --- a/drivers/gpu/drm/i915/gvt/opregion.c +++ b/drivers/gpu/drm/i915/gvt/opregion.c @@ -73,7 +73,7 @@ static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map) } ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, vgpu_opregion(vgpu)->gfn[i], - mfn, 1, map, GVT_MAP_OPREGION); + mfn, 1, map); if (ret) { gvt_err("fail to map GFN to MFN, errno: %d\n", ret); return ret; @@ -89,28 +89,18 @@ static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map) */ void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu) { - int i; - gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id); if (!vgpu_opregion(vgpu)->va) return; - if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) { - vunmap(vgpu_opregion(vgpu)->va); - for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) { - if (vgpu_opregion(vgpu)->pages[i]) { - put_page(vgpu_opregion(vgpu)->pages[i]); - vgpu_opregion(vgpu)->pages[i] = NULL; - } - } - } else { + if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) { map_vgpu_opregion(vgpu, false); free_pages((unsigned long)vgpu_opregion(vgpu)->va, INTEL_GVT_OPREGION_PORDER); - } - vgpu_opregion(vgpu)->va = NULL; + vgpu_opregion(vgpu)->va = NULL; + } } /** @@ -137,22 +127,8 @@ int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa) ret = map_vgpu_opregion(vgpu, true); if (ret) return ret; - } else { - gvt_dbg_core("emulate opregion from userspace\n"); - - /* - * If opregion pages are not allocated from host kenrel, - * most of the params are meaningless - */ - ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, - 0, /* not used */ - 0, /* not used */ - 2, /* not used */ - 1, - GVT_MAP_OPREGION); - if (ret) - return ret; } + return 0; } @@ -163,7 +139,7 @@ int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa) */ void intel_gvt_clean_opregion(struct intel_gvt *gvt) { - iounmap(gvt->opregion.opregion_va); + memunmap(gvt->opregion.opregion_va); gvt->opregion.opregion_va = NULL; } @@ -181,8 +157,8 @@ int intel_gvt_init_opregion(struct intel_gvt *gvt) pci_read_config_dword(gvt->dev_priv->drm.pdev, INTEL_GVT_PCI_OPREGION, &gvt->opregion.opregion_pa); - gvt->opregion.opregion_va = acpi_os_ioremap(gvt->opregion.opregion_pa, - INTEL_GVT_OPREGION_SIZE); + gvt->opregion.opregion_va = memremap(gvt->opregion.opregion_pa, + INTEL_GVT_OPREGION_SIZE, MEMREMAP_WB); if (!gvt->opregion.opregion_va) { gvt_err("fail to map host opregion\n"); return -EFAULT; diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c index feebb65ba641..44136b1f3aab 100644 --- a/drivers/gpu/drm/i915/gvt/render.c +++ b/drivers/gpu/drm/i915/gvt/render.c @@ -118,6 +118,7 @@ static u32 gen9_render_mocs_L3[32]; static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) { struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + enum forcewake_domains fw; i915_reg_t reg; u32 regs[] = { [RCS] = 0x4260, @@ -135,10 +136,26 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) reg = _MMIO(regs[ring_id]); - I915_WRITE(reg, 0x1); + /* WaForceWakeRenderDuringMmioTLBInvalidate:skl + * we need to put a forcewake when invalidating RCS TLB caches, + * otherwise device can go to RC6 state and interrupt invalidation + * process + */ + fw = intel_uncore_forcewake_for_reg(dev_priv, reg, + FW_REG_READ | FW_REG_WRITE); + if (ring_id == RCS && IS_SKYLAKE(dev_priv)) + fw |= FORCEWAKE_RENDER; - if (wait_for_atomic((I915_READ(reg) == 0), 50)) + intel_uncore_forcewake_get(dev_priv, fw); + + I915_WRITE_FW(reg, 0x1); + + if (wait_for_atomic((I915_READ_FW(reg) == 0), 50)) gvt_err("timeout in invalidate ring (%d) tlb\n", ring_id); + else + vgpu_vreg(vgpu, regs[ring_id]) = 0; + + intel_uncore_forcewake_put(dev_priv, fw); gvt_dbg_core("invalidate TLB for ring %d\n", ring_id); } @@ -162,6 +179,7 @@ static void load_mocs(struct intel_vgpu *vgpu, int ring_id) if (!IS_SKYLAKE(dev_priv)) return; + offset.reg = regs[ring_id]; for (i = 0; i < 64; i++) { gen9_render_mocs[ring_id][i] = I915_READ(offset); I915_WRITE(offset, vgpu_vreg(vgpu, offset)); @@ -199,6 +217,7 @@ static void restore_mocs(struct intel_vgpu *vgpu, int ring_id) if (!IS_SKYLAKE(dev_priv)) return; + offset.reg = regs[ring_id]; for (i = 0; i < 64; i++) { vgpu_vreg(vgpu, offset) = I915_READ(offset); I915_WRITE(offset, gen9_render_mocs[ring_id][i]); diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c index 1df6a5460f3e..678b0be85376 100644 --- a/drivers/gpu/drm/i915/gvt/sched_policy.c +++ b/drivers/gpu/drm/i915/gvt/sched_policy.c @@ -36,12 +36,10 @@ static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu) { - struct intel_vgpu_execlist *execlist; enum intel_engine_id i; struct intel_engine_cs *engine; for_each_engine(engine, vgpu->gvt->dev_priv, i) { - execlist = &vgpu->execlist[i]; if (!list_empty(workload_q_head(vgpu, i))) return true; } diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index e96eaeebeb0a..4db242250235 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -89,15 +89,15 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload) } page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i); - dst = kmap_atomic(page); + dst = kmap(page); intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst, GTT_PAGE_SIZE); - kunmap_atomic(dst); + kunmap(page); i++; } page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); - shadow_ring_context = kmap_atomic(page); + shadow_ring_context = kmap(page); #define COPY_REG(name) \ intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ @@ -123,7 +123,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload) sizeof(*shadow_ring_context), GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); - kunmap_atomic(shadow_ring_context); + kunmap(page); return 0; } @@ -160,8 +160,6 @@ static int shadow_context_status_change(struct notifier_block *nb, static int dispatch_workload(struct intel_vgpu_workload *workload) { - struct intel_vgpu *vgpu = workload->vgpu; - struct intel_gvt *gvt = vgpu->gvt; int ring_id = workload->ring_id; struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; @@ -174,51 +172,49 @@ static int dispatch_workload(struct intel_vgpu_workload *workload) shadow_ctx->desc_template = workload->ctx_desc.addressing_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT; + mutex_lock(&dev_priv->drm.struct_mutex); + rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx); if (IS_ERR(rq)) { gvt_err("fail to allocate gem request\n"); - workload->status = PTR_ERR(rq); - return workload->status; + ret = PTR_ERR(rq); + goto out; } gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq); workload->req = i915_gem_request_get(rq); - mutex_lock(&gvt->lock); - ret = intel_gvt_scan_and_shadow_workload(workload); if (ret) - goto err; + goto out; ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx); if (ret) - goto err; + goto out; ret = populate_shadow_context(workload); if (ret) - goto err; + goto out; if (workload->prepare) { ret = workload->prepare(workload); if (ret) - goto err; + goto out; } - mutex_unlock(&gvt->lock); - gvt_dbg_sched("ring id %d submit workload to i915 %p\n", ring_id, workload->req); - i915_add_request_no_flush(rq); + ret = 0; workload->dispatched = true; - return 0; -err: - workload->status = ret; - - mutex_unlock(&gvt->lock); +out: + if (ret) + workload->status = ret; - i915_add_request_no_flush(rq); + if (!IS_ERR_OR_NULL(rq)) + i915_add_request_no_flush(rq); + mutex_unlock(&dev_priv->drm.struct_mutex); return ret; } @@ -318,10 +314,10 @@ static void update_guest_context(struct intel_vgpu_workload *workload) } page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i); - src = kmap_atomic(page); + src = kmap(page); intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src, GTT_PAGE_SIZE); - kunmap_atomic(src); + kunmap(page); i++; } @@ -329,7 +325,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload) RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4); page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); - shadow_ring_context = kmap_atomic(page); + shadow_ring_context = kmap(page); #define COPY_REG(name) \ intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \ @@ -347,7 +343,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload) sizeof(*shadow_ring_context), GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); - kunmap_atomic(shadow_ring_context); + kunmap(page); } static void complete_current_workload(struct intel_gvt *gvt, int ring_id) @@ -400,21 +396,27 @@ static int workload_thread(void *priv) int ring_id = p->ring_id; struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; struct intel_vgpu_workload *workload = NULL; + long lret; int ret; bool need_force_wake = IS_SKYLAKE(gvt->dev_priv); + DEFINE_WAIT_FUNC(wait, woken_wake_function); kfree(p); gvt_dbg_core("workload thread for ring %d started\n", ring_id); while (!kthread_should_stop()) { - ret = wait_event_interruptible(scheduler->waitq[ring_id], - kthread_should_stop() || - (workload = pick_next_workload(gvt, ring_id))); - - WARN_ON_ONCE(ret); - - if (kthread_should_stop()) + add_wait_queue(&scheduler->waitq[ring_id], &wait); + do { + workload = pick_next_workload(gvt, ring_id); + if (workload) + break; + wait_woken(&wait, TASK_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT); + } while (!kthread_should_stop()); + remove_wait_queue(&scheduler->waitq[ring_id], &wait); + + if (!workload) break; mutex_lock(&scheduler_mutex); @@ -432,9 +434,9 @@ static int workload_thread(void *priv) intel_uncore_forcewake_get(gvt->dev_priv, FORCEWAKE_ALL); - mutex_lock(&gvt->dev_priv->drm.struct_mutex); + mutex_lock(&gvt->lock); ret = dispatch_workload(workload); - mutex_unlock(&gvt->dev_priv->drm.struct_mutex); + mutex_unlock(&gvt->lock); if (ret) { gvt_err("fail to dispatch workload, skip\n"); @@ -444,20 +446,23 @@ static int workload_thread(void *priv) gvt_dbg_sched("ring id %d wait workload %p\n", workload->ring_id, workload); - workload->status = i915_wait_request(workload->req, - 0, NULL, NULL); - if (workload->status != 0) + lret = i915_wait_request(workload->req, + 0, MAX_SCHEDULE_TIMEOUT); + if (lret < 0) { + workload->status = lret; gvt_err("fail to wait workload, skip\n"); + } else { + workload->status = 0; + } complete: gvt_dbg_sched("will complete workload %p\n, status: %d\n", workload, workload->status); - mutex_lock(&gvt->dev_priv->drm.struct_mutex); complete_current_workload(gvt, ring_id); - mutex_unlock(&gvt->dev_priv->drm.struct_mutex); - i915_gem_request_put(fetch_and_zero(&workload->req)); + if (workload->req) + i915_gem_request_put(fetch_and_zero(&workload->req)); if (need_force_wake) intel_uncore_forcewake_put(gvt->dev_priv, diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 9401436d721f..536d2b9d5777 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -41,14 +41,18 @@ static void clean_vgpu_mmio(struct intel_vgpu *vgpu) vgpu->mmio.vreg = vgpu->mmio.sreg = NULL; } -static int setup_vgpu_mmio(struct intel_vgpu *vgpu) +int setup_vgpu_mmio(struct intel_vgpu *vgpu) { struct intel_gvt *gvt = vgpu->gvt; const struct intel_gvt_device_info *info = &gvt->device_info; - vgpu->mmio.vreg = vzalloc(info->mmio_size * 2); - if (!vgpu->mmio.vreg) - return -ENOMEM; + if (vgpu->mmio.vreg) + memset(vgpu->mmio.vreg, 0, info->mmio_size * 2); + else { + vgpu->mmio.vreg = vzalloc(info->mmio_size * 2); + if (!vgpu->mmio.vreg) + return -ENOMEM; + } vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size; @@ -95,6 +99,7 @@ static void setup_vgpu_cfg_space(struct intel_vgpu *vgpu, */ memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4); memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4); + memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4); for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) { vgpu->cfg_space.bar[i].size = pci_resource_len( @@ -103,7 +108,7 @@ static void setup_vgpu_cfg_space(struct intel_vgpu *vgpu, } } -static void populate_pvinfo_page(struct intel_vgpu *vgpu) +void populate_pvinfo_page(struct intel_vgpu *vgpu) { /* setup the ballooning information */ vgpu_vreg64(vgpu, vgtif_reg(magic)) = VGT_MAGIC; @@ -133,6 +138,106 @@ static void populate_pvinfo_page(struct intel_vgpu *vgpu) } /** + * intel_gvt_init_vgpu_types - initialize vGPU type list + * @gvt : GVT device + * + * Initialize vGPU type list based on available resource. + * + */ +int intel_gvt_init_vgpu_types(struct intel_gvt *gvt) +{ + unsigned int num_types; + unsigned int i, low_avail; + unsigned int min_low; + + /* vGPU type name is defined as GVTg_Vx_y which contains + * physical GPU generation type and 'y' means maximum vGPU + * instances user can create on one physical GPU for this + * type. + * + * Depend on physical SKU resource, might see vGPU types like + * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create + * different types of vGPU on same physical GPU depending on + * available resource. Each vGPU type will have "avail_instance" + * to indicate how many vGPU instance can be created for this + * type. + * + * Currently use static size here as we init type earlier.. + */ + low_avail = MB_TO_BYTES(256) - HOST_LOW_GM_SIZE; + num_types = 4; + + gvt->types = kzalloc(num_types * sizeof(struct intel_vgpu_type), + GFP_KERNEL); + if (!gvt->types) + return -ENOMEM; + + min_low = MB_TO_BYTES(32); + for (i = 0; i < num_types; ++i) { + if (low_avail / min_low == 0) + break; + gvt->types[i].low_gm_size = min_low; + gvt->types[i].high_gm_size = 3 * gvt->types[i].low_gm_size; + gvt->types[i].fence = 4; + gvt->types[i].max_instance = low_avail / min_low; + gvt->types[i].avail_instance = gvt->types[i].max_instance; + + if (IS_GEN8(gvt->dev_priv)) + sprintf(gvt->types[i].name, "GVTg_V4_%u", + gvt->types[i].max_instance); + else if (IS_GEN9(gvt->dev_priv)) + sprintf(gvt->types[i].name, "GVTg_V5_%u", + gvt->types[i].max_instance); + + min_low <<= 1; + gvt_dbg_core("type[%d]: %s max %u avail %u low %u high %u fence %u\n", + i, gvt->types[i].name, gvt->types[i].max_instance, + gvt->types[i].avail_instance, + gvt->types[i].low_gm_size, + gvt->types[i].high_gm_size, gvt->types[i].fence); + } + + gvt->num_types = i; + return 0; +} + +void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt) +{ + kfree(gvt->types); +} + +static void intel_gvt_update_vgpu_types(struct intel_gvt *gvt) +{ + int i; + unsigned int low_gm_avail, high_gm_avail, fence_avail; + unsigned int low_gm_min, high_gm_min, fence_min, total_min; + + /* Need to depend on maxium hw resource size but keep on + * static config for now. + */ + low_gm_avail = MB_TO_BYTES(256) - HOST_LOW_GM_SIZE - + gvt->gm.vgpu_allocated_low_gm_size; + high_gm_avail = MB_TO_BYTES(256) * 3 - HOST_HIGH_GM_SIZE - + gvt->gm.vgpu_allocated_high_gm_size; + fence_avail = gvt_fence_sz(gvt) - HOST_FENCE - + gvt->fence.vgpu_allocated_fence_num; + + for (i = 0; i < gvt->num_types; i++) { + low_gm_min = low_gm_avail / gvt->types[i].low_gm_size; + high_gm_min = high_gm_avail / gvt->types[i].high_gm_size; + fence_min = fence_avail / gvt->types[i].fence; + total_min = min(min(low_gm_min, high_gm_min), fence_min); + gvt->types[i].avail_instance = min(gvt->types[i].max_instance, + total_min); + + gvt_dbg_core("update type[%d]: %s max %u avail %u low %u high %u fence %u\n", + i, gvt->types[i].name, gvt->types[i].max_instance, + gvt->types[i].avail_instance, gvt->types[i].low_gm_size, + gvt->types[i].high_gm_size, gvt->types[i].fence); + } +} + +/** * intel_gvt_destroy_vgpu - destroy a virtual GPU * @vgpu: virtual GPU * @@ -166,20 +271,11 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu) clean_vgpu_mmio(vgpu); vfree(vgpu); + intel_gvt_update_vgpu_types(gvt); mutex_unlock(&gvt->lock); } -/** - * intel_gvt_create_vgpu - create a virtual GPU - * @gvt: GVT device - * @param: vGPU creation parameters - * - * This function is called when user wants to create a virtual GPU. - * - * Returns: - * pointer to intel_vgpu, error pointer if failed. - */ -struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, +static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, struct intel_vgpu_creation_params *param) { struct intel_vgpu *vgpu; @@ -224,15 +320,9 @@ struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, if (ret) goto out_detach_hypervisor_vgpu; - if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) { - ret = intel_vgpu_init_opregion(vgpu, 0); - if (ret) - goto out_clean_gtt; - } - ret = intel_vgpu_init_display(vgpu); if (ret) - goto out_clean_opregion; + goto out_clean_gtt; ret = intel_vgpu_init_execlist(vgpu); if (ret) @@ -257,8 +347,6 @@ out_clean_execlist: intel_vgpu_clean_execlist(vgpu); out_clean_display: intel_vgpu_clean_display(vgpu); -out_clean_opregion: - intel_vgpu_clean_opregion(vgpu); out_clean_gtt: intel_vgpu_clean_gtt(vgpu); out_detach_hypervisor_vgpu: @@ -272,3 +360,50 @@ out_free_vgpu: mutex_unlock(&gvt->lock); return ERR_PTR(ret); } + +/** + * intel_gvt_create_vgpu - create a virtual GPU + * @gvt: GVT device + * @type: type of the vGPU to create + * + * This function is called when user wants to create a virtual GPU. + * + * Returns: + * pointer to intel_vgpu, error pointer if failed. + */ +struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, + struct intel_vgpu_type *type) +{ + struct intel_vgpu_creation_params param; + struct intel_vgpu *vgpu; + + param.handle = 0; + param.primary = 1; + param.low_gm_sz = type->low_gm_size; + param.high_gm_sz = type->high_gm_size; + param.fence_sz = type->fence; + + /* XXX current param based on MB */ + param.low_gm_sz = BYTES_TO_MB(param.low_gm_sz); + param.high_gm_sz = BYTES_TO_MB(param.high_gm_sz); + + vgpu = __intel_gvt_create_vgpu(gvt, ¶m); + if (IS_ERR(vgpu)) + return vgpu; + + /* calculate left instance change for types */ + intel_gvt_update_vgpu_types(gvt); + + return vgpu; +} + +/** + * intel_gvt_reset_vgpu - reset a virtual GPU + * @vgpu: virtual GPU + * + * This function is called when user wants to reset a virtual GPU. + * + */ +void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu) +{ +} diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index f191d7b66b1d..f5039f4f988f 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, } if (ret == 0 && needs_clflush_after) - drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len); + drm_clflush_virt_range(shadow_batch_obj->mm.mapping, batch_len); i915_gem_object_unpin_map(shadow_batch_obj); return ret; diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 20638d22bbad..791bfc760075 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -107,12 +107,12 @@ static char get_tiling_flag(struct drm_i915_gem_object *obj) static char get_global_flag(struct drm_i915_gem_object *obj) { - return obj->fault_mappable ? 'g' : ' '; + return !list_empty(&obj->userfault_link) ? 'g' : ' '; } static char get_pin_mapped_flag(struct drm_i915_gem_object *obj) { - return obj->mapping ? 'M' : ' '; + return obj->mm.mapping ? 'M' : ' '; } static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj) @@ -136,11 +136,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) struct i915_vma *vma; unsigned int frontbuffer_bits; int pin_count = 0; - enum intel_engine_id id; lockdep_assert_held(&obj->base.dev->struct_mutex); - seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x [ ", + seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x %s%s%s", &obj->base, get_active_flag(obj), get_pin_flag(obj), @@ -149,17 +148,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) get_pin_mapped_flag(obj), obj->base.size / 1024, obj->base.read_domains, - obj->base.write_domain); - for_each_engine(engine, dev_priv, id) - seq_printf(m, "%x ", - i915_gem_active_get_seqno(&obj->last_read[id], - &obj->base.dev->struct_mutex)); - seq_printf(m, "] %x %s%s%s", - i915_gem_active_get_seqno(&obj->last_write, - &obj->base.dev->struct_mutex), + obj->base.write_domain, i915_cache_level_str(dev_priv, obj->cache_level), - obj->dirty ? " dirty" : "", - obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); + obj->mm.dirty ? " dirty" : "", + obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : ""); if (obj->base.name) seq_printf(m, " (name: %d)", obj->base.name); list_for_each_entry(vma, &obj->vma_list, obj_link) { @@ -187,8 +179,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) if (obj->stolen) seq_printf(m, " (stolen: %08llx)", obj->stolen->start); - engine = i915_gem_active_get_engine(&obj->last_write, - &dev_priv->drm.struct_mutex); + engine = i915_gem_object_last_write_engine(obj); if (engine) seq_printf(m, " (%s)", engine->name); @@ -226,7 +217,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) return ret; total_obj_size = total_gtt_size = count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (obj->stolen == NULL) continue; @@ -236,7 +227,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) total_gtt_size += i915_gem_obj_total_ggtt_size(obj); count++; } - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { if (obj->stolen == NULL) continue; @@ -399,16 +390,16 @@ static int i915_gem_object_info(struct seq_file *m, void *data) size = count = 0; mapped_size = mapped_count = 0; purgeable_size = purgeable_count = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { size += obj->base.size; ++count; - if (obj->madv == I915_MADV_DONTNEED) { + if (obj->mm.madv == I915_MADV_DONTNEED) { purgeable_size += obj->base.size; ++purgeable_count; } - if (obj->mapping) { + if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; } @@ -416,7 +407,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) seq_printf(m, "%u unbound objects, %llu bytes\n", count, size); size = count = dpy_size = dpy_count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { size += obj->base.size; ++count; @@ -425,12 +416,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) ++dpy_count; } - if (obj->madv == I915_MADV_DONTNEED) { + if (obj->mm.madv == I915_MADV_DONTNEED) { purgeable_size += obj->base.size; ++purgeable_count; } - if (obj->mapping) { + if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; } @@ -502,7 +493,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) return ret; total_obj_size = total_gtt_size = count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (show_pin_display_only && !obj->pin_display) continue; @@ -556,12 +547,12 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) pipe, plane); } if (work->flip_queued_req) { - struct intel_engine_cs *engine = i915_gem_request_get_engine(work->flip_queued_req); + struct intel_engine_cs *engine = work->flip_queued_req->engine; seq_printf(m, "Flip queued on %s at seqno %x, next seqno %x [current breadcrumb %x], completed? %d\n", engine->name, - i915_gem_request_get_seqno(work->flip_queued_req), - dev_priv->next_seqno, + work->flip_queued_req->global_seqno, + atomic_read(&dev_priv->gt.global_timeline.next_seqno), intel_engine_get_seqno(engine), i915_gem_request_completed(work->flip_queued_req)); } else @@ -640,17 +631,11 @@ static void print_request(struct seq_file *m, struct drm_i915_gem_request *rq, const char *prefix) { - struct pid *pid = rq->ctx->pid; - struct task_struct *task; - - rcu_read_lock(); - task = pid ? pid_task(pid, PIDTYPE_PID) : NULL; - seq_printf(m, "%s%x [%x:%x] @ %d: %s [%d]\n", prefix, - rq->fence.seqno, rq->ctx->hw_id, rq->fence.seqno, + seq_printf(m, "%s%x [%x:%x] prio=%d @ %dms: %s\n", prefix, + rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno, + rq->priotree.priority, jiffies_to_msecs(jiffies - rq->emitted_jiffies), - task ? task->comm : "<unknown>", - task ? task->pid : -1); - rcu_read_unlock(); + rq->timeline->common->name); } static int i915_gem_request_info(struct seq_file *m, void *data) @@ -671,13 +656,13 @@ static int i915_gem_request_info(struct seq_file *m, void *data) int count; count = 0; - list_for_each_entry(req, &engine->request_list, link) + list_for_each_entry(req, &engine->timeline->requests, link) count++; if (count == 0) continue; seq_printf(m, "%s requests: %d\n", engine->name, count); - list_for_each_entry(req, &engine->request_list, link) + list_for_each_entry(req, &engine->timeline->requests, link) print_request(m, req, " "); any++; @@ -699,14 +684,14 @@ static void i915_ring_seqno_info(struct seq_file *m, seq_printf(m, "Current sequence (%s): %x\n", engine->name, intel_engine_get_seqno(engine)); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "Waiting (%s): %s [%d] on %x\n", engine->name, w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static int i915_gem_seqno_info(struct seq_file *m, void *data) @@ -743,17 +728,32 @@ static int i915_interrupt_info(struct seq_file *m, void *data) I915_READ(VLV_IIR_RW)); seq_printf(m, "Display IMR:\t%08x\n", I915_READ(VLV_IMR)); - for_each_pipe(dev_priv, pipe) + for_each_pipe(dev_priv, pipe) { + enum intel_display_power_domain power_domain; + + power_domain = POWER_DOMAIN_PIPE(pipe); + if (!intel_display_power_get_if_enabled(dev_priv, + power_domain)) { + seq_printf(m, "Pipe %c power disabled\n", + pipe_name(pipe)); + continue; + } + seq_printf(m, "Pipe %c stat:\t%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); + intel_display_power_put(dev_priv, power_domain); + } + + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); seq_printf(m, "Port hotplug:\t%08x\n", I915_READ(PORT_HOTPLUG_EN)); seq_printf(m, "DPFLIPSTAT:\t%08x\n", I915_READ(VLV_DPFLIPSTAT)); seq_printf(m, "DPINVGTT:\t%08x\n", I915_READ(DPINVGTT)); + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); for (i = 0; i < 4; i++) { seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", @@ -935,27 +935,6 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data) return 0; } -static int i915_hws_info(struct seq_file *m, void *data) -{ - struct drm_info_node *node = m->private; - struct drm_i915_private *dev_priv = node_to_i915(node); - struct intel_engine_cs *engine; - const u32 *hws; - int i; - - engine = dev_priv->engine[(uintptr_t)node->info_ent->data]; - hws = engine->status_page.page_addr; - if (hws == NULL) - return 0; - - for (i = 0; i < 4096 / sizeof(u32) / 4; i += 4) { - seq_printf(m, "0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", - i * 4, - hws[i], hws[i + 1], hws[i + 2], hws[i + 3]); - } - return 0; -} - #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) static ssize_t @@ -1046,15 +1025,8 @@ static int i915_next_seqno_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - int ret; - - ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex); - if (ret) - return ret; - - *val = dev_priv->next_seqno; - mutex_unlock(&dev_priv->drm.struct_mutex); + *val = 1 + atomic_read(&dev_priv->gt.global_timeline.next_seqno); return 0; } @@ -1069,7 +1041,7 @@ i915_next_seqno_set(void *data, u64 val) if (ret) return ret; - ret = i915_gem_set_seqno(dev, val); + ret = i915_gem_set_global_seqno(dev, val); mutex_unlock(&dev->struct_mutex); return ret; @@ -1356,21 +1328,20 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) seq_printf(m, "%s:\n", engine->name); seq_printf(m, "\tseqno = %x [current %x, last %x]\n", - engine->hangcheck.seqno, - seqno[id], - engine->last_submitted_seqno); + engine->hangcheck.seqno, seqno[id], + intel_engine_last_submit(engine)); seq_printf(m, "\twaiters? %s, fake irq active? %s\n", yesno(intel_engine_has_waiter(engine)), yesno(test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings))); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "\t%s [%d] waiting for %x\n", w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n", (long long)engine->hangcheck.acthd, @@ -1396,14 +1367,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) static int ironlake_drpc_info(struct seq_file *m) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; u32 rgvmodectl, rstdbyctl; u16 crstandvid; - int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); rgvmodectl = I915_READ(MEMMODECTL); @@ -1411,7 +1377,6 @@ static int ironlake_drpc_info(struct seq_file *m) crstandvid = I915_READ16(CRSTANDVID); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN)); seq_printf(m, "Boost freq: %d\n", @@ -1674,11 +1639,13 @@ static int i915_fbc_status(struct seq_file *m, void *unused) seq_printf(m, "FBC disabled: %s\n", dev_priv->fbc.no_fbc_reason); - if (intel_fbc_is_active(dev_priv) && - INTEL_GEN(dev_priv) >= 7) + if (intel_fbc_is_active(dev_priv) && INTEL_GEN(dev_priv) >= 7) { + uint32_t mask = INTEL_GEN(dev_priv) >= 8 ? + BDW_FBC_COMPRESSION_MASK : + IVB_FBC_COMPRESSION_MASK; seq_printf(m, "Compressing: %s\n", - yesno(I915_READ(FBC_STATUS2) & - FBC_COMPRESSION_MASK)); + yesno(I915_READ(FBC_STATUS2) & mask)); + } mutex_unlock(&dev_priv->fbc.lock); intel_runtime_pm_put(dev_priv); @@ -1757,6 +1724,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) bool sr_enabled = false; intel_runtime_pm_get(dev_priv); + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); if (HAS_PCH_SPLIT(dev_priv)) sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN; @@ -1770,10 +1738,10 @@ static int i915_sr_status(struct seq_file *m, void *unused) else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); intel_runtime_pm_put(dev_priv); - seq_printf(m, "self-refresh: %s\n", - sr_enabled ? "enabled" : "disabled"); + seq_printf(m, "self-refresh: %s\n", enableddisabled(sr_enabled)); return 0; } @@ -1907,7 +1875,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) fbdev_fb->base.height, fbdev_fb->base.depth, fbdev_fb->base.bits_per_pixel, - fbdev_fb->base.modifier[0], + fbdev_fb->base.modifier, drm_framebuffer_read_refcount(&fbdev_fb->base)); describe_obj(m, fbdev_fb->obj); seq_putc(m, '\n'); @@ -1925,7 +1893,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) fb->base.height, fb->base.depth, fb->base.bits_per_pixel, - fb->base.modifier[0], + fb->base.modifier, drm_framebuffer_read_refcount(&fb->base)); describe_obj(m, fb->obj); seq_putc(m, '\n'); @@ -2015,7 +1983,7 @@ static void i915_dump_lrc_obj(struct seq_file *m, seq_printf(m, "\tBound in GGTT at 0x%08x\n", i915_ggtt_offset(vma)); - if (i915_gem_object_get_pages(vma->obj)) { + if (i915_gem_object_pin_pages(vma->obj)) { seq_puts(m, "\tFailed to get pages for context object\n\n"); return; } @@ -2034,6 +2002,7 @@ static void i915_dump_lrc_obj(struct seq_file *m, kunmap_atomic(reg_state); } + i915_gem_object_unpin_pages(vma->obj); seq_putc(m, '\n'); } @@ -2091,12 +2060,7 @@ static const char *swizzle_string(unsigned swizzle) static int i915_swizzle_info(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; - int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); seq_printf(m, "bit6 swizzle for X-tiling = %s\n", @@ -2136,7 +2100,6 @@ static int i915_swizzle_info(struct seq_file *m, void *data) seq_puts(m, "L-shaped memory detected\n"); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); return 0; } @@ -2292,8 +2255,8 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) struct drm_file *file; seq_printf(m, "RPS enabled? %d\n", dev_priv->rps.enabled); - seq_printf(m, "GPU busy? %s [%x]\n", - yesno(dev_priv->gt.awake), dev_priv->gt.active_engines); + seq_printf(m, "GPU busy? %s [%d requests]\n", + yesno(dev_priv->gt.awake), dev_priv->gt.active_requests); seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv)); seq_printf(m, "Frequency requested %d\n", intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq)); @@ -2328,7 +2291,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) if (INTEL_GEN(dev_priv) >= 6 && dev_priv->rps.enabled && - dev_priv->gt.active_engines) { + dev_priv->gt.active_requests) { u32 rpup, rpupei; u32 rpdown, rpdownei; @@ -2409,6 +2372,32 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data) return 0; } +static void i915_guc_log_info(struct seq_file *m, + struct drm_i915_private *dev_priv) +{ + struct intel_guc *guc = &dev_priv->guc; + + seq_puts(m, "\nGuC logging stats:\n"); + + seq_printf(m, "\tISR: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_ISR_LOG_BUFFER], + guc->log.total_overflow_count[GUC_ISR_LOG_BUFFER]); + + seq_printf(m, "\tDPC: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_DPC_LOG_BUFFER], + guc->log.total_overflow_count[GUC_DPC_LOG_BUFFER]); + + seq_printf(m, "\tCRASH: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_CRASH_DUMP_LOG_BUFFER], + guc->log.total_overflow_count[GUC_CRASH_DUMP_LOG_BUFFER]); + + seq_printf(m, "\tTotal flush interrupt count: %u\n", + guc->log.flush_interrupt_count); + + seq_printf(m, "\tCapture miss count: %u\n", + guc->log.capture_miss_count); +} + static void i915_guc_client_info(struct seq_file *m, struct drm_i915_private *dev_priv, struct i915_guc_client *client) @@ -2482,6 +2471,8 @@ static int i915_guc_info(struct seq_file *m, void *data) seq_printf(m, "\nGuC execbuf client @ %p:\n", guc.execbuf_client); i915_guc_client_info(m, dev_priv, &client); + i915_guc_log_info(m, dev_priv); + /* Add more as required ... */ return 0; @@ -2493,10 +2484,10 @@ static int i915_guc_log_dump(struct seq_file *m, void *data) struct drm_i915_gem_object *obj; int i = 0, pg; - if (!dev_priv->guc.log_vma) + if (!dev_priv->guc.log.vma) return 0; - obj = dev_priv->guc.log_vma->obj; + obj = dev_priv->guc.log.vma->obj; for (pg = 0; pg < obj->base.size / PAGE_SIZE; pg++) { u32 *log = kmap_atomic(i915_gem_object_get_page(obj, pg)); @@ -2513,6 +2504,44 @@ static int i915_guc_log_dump(struct seq_file *m, void *data) return 0; } +static int i915_guc_log_control_get(void *data, u64 *val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = to_i915(dev); + + if (!dev_priv->guc.log.vma) + return -EINVAL; + + *val = i915.guc_log_level; + + return 0; +} + +static int i915_guc_log_control_set(void *data, u64 val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = to_i915(dev); + int ret; + + if (!dev_priv->guc.log.vma) + return -EINVAL; + + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; + + intel_runtime_pm_get(dev_priv); + ret = i915_guc_log_control(dev_priv, val); + intel_runtime_pm_put(dev_priv); + + mutex_unlock(&dev->struct_mutex); + return ret; +} + +DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_control_fops, + i915_guc_log_control_get, i915_guc_log_control_set, + "%lld\n"); + static int i915_edp_psr_status(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); @@ -2542,11 +2571,22 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE; else { for_each_pipe(dev_priv, pipe) { + enum transcoder cpu_transcoder = + intel_pipe_to_cpu_transcoder(dev_priv, pipe); + enum intel_display_power_domain power_domain; + + power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); + if (!intel_display_power_get_if_enabled(dev_priv, + power_domain)) + continue; + stat[pipe] = I915_READ(VLV_PSRSTAT(pipe)) & VLV_EDP_PSR_CURR_STATE_MASK; if ((stat[pipe] == VLV_EDP_PSR_ACTIVE_NORFB_UP) || (stat[pipe] == VLV_EDP_PSR_ACTIVE_SF_UPDATE)) enabled = true; + + intel_display_power_put(dev_priv, power_domain); } } @@ -2971,7 +3011,7 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { struct drm_plane_state *state; struct drm_plane *plane = &intel_plane->base; - char *format_name; + struct drm_format_name_buf format_name; if (!plane->state) { seq_puts(m, "plane->state is NULL!\n"); @@ -2981,9 +3021,9 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) state = plane->state; if (state->fb) { - format_name = drm_get_format_name(state->fb->pixel_format); + drm_get_format_name(state->fb->pixel_format, &format_name); } else { - format_name = kstrdup("N/A", GFP_KERNEL); + sprintf(format_name.str, "N/A"); } seq_printf(m, "\t--Plane id %d: type=%s, crtc_pos=%4dx%4d, crtc_size=%4dx%4d, src_pos=%d.%04ux%d.%04u, src_size=%d.%04ux%d.%04u, format=%s, rotation=%s\n", @@ -2999,10 +3039,8 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) ((state->src_w & 0xffff) * 15625) >> 10, (state->src_h >> 16), ((state->src_h & 0xffff) * 15625) >> 10, - format_name, + format_name.str, plane_rotation(state->rotation)); - - kfree(format_name); } } @@ -3094,6 +3132,8 @@ static int i915_engine_info(struct seq_file *m, void *unused) struct intel_engine_cs *engine; enum intel_engine_id id; + intel_runtime_pm_get(dev_priv); + for_each_engine(engine, dev_priv, id) { struct intel_breadcrumbs *b = &engine->breadcrumbs; struct drm_i915_gem_request *rq; @@ -3103,7 +3143,7 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_printf(m, "%s\n", engine->name); seq_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [score %d]\n", intel_engine_get_seqno(engine), - engine->last_submitted_seqno, + intel_engine_last_submit(engine), engine->hangcheck.seqno, engine->hangcheck.score); @@ -3111,14 +3151,14 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_printf(m, "\tRequests:\n"); - rq = list_first_entry(&engine->request_list, - struct drm_i915_gem_request, link); - if (&rq->link != &engine->request_list) + rq = list_first_entry(&engine->timeline->requests, + struct drm_i915_gem_request, link); + if (&rq->link != &engine->timeline->requests) print_request(m, rq, "\t\tfirst "); - rq = list_last_entry(&engine->request_list, - struct drm_i915_gem_request, link); - if (&rq->link != &engine->request_list) + rq = list_last_entry(&engine->timeline->requests, + struct drm_i915_gem_request, link); + if (&rq->link != &engine->timeline->requests) print_request(m, rq, "\t\tlast "); rq = i915_gem_find_active_request(engine); @@ -3155,6 +3195,7 @@ static int i915_engine_info(struct seq_file *m, void *unused) if (i915.enable_execlists) { u32 ptr, read, write; + struct rb_node *rb; seq_printf(m, "\tExeclist status: 0x%08x %08x\n", I915_READ(RING_EXECLIST_STATUS_LO(engine)), @@ -3192,6 +3233,13 @@ static int i915_engine_info(struct seq_file *m, void *unused) else seq_printf(m, "\t\tELSP[1] idle\n"); rcu_read_unlock(); + + spin_lock_irq(&engine->timeline->lock); + for (rb = engine->execlist_first; rb; rb = rb_next(rb)) { + rq = rb_entry(rb, typeof(*rq), priotree.node); + print_request(m, rq, "\t\tQ "); + } + spin_unlock_irq(&engine->timeline->lock); } else if (INTEL_GEN(dev_priv) > 6) { seq_printf(m, "\tPP_DIR_BASE: 0x%08x\n", I915_READ(RING_PP_DIR_BASE(engine))); @@ -3201,18 +3249,20 @@ static int i915_engine_info(struct seq_file *m, void *unused) I915_READ(RING_PP_DIR_DCLV(engine))); } - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "\t%s [%d] waiting for %x\n", w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); seq_puts(m, "\n"); } + intel_runtime_pm_put(dev_priv); + return 0; } @@ -3274,15 +3324,6 @@ static int i915_semaphore_status(struct seq_file *m, void *unused) seq_putc(m, '\n'); } - seq_puts(m, "\nSync seqno:\n"); - for_each_engine(engine, dev_priv, id) { - for (j = 0; j < num_rings; j++) - seq_printf(m, " 0x%08x ", - engine->semaphore.sync_seqno[j]); - seq_putc(m, '\n'); - } - seq_putc(m, '\n'); - intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); return 0; @@ -3375,7 +3416,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused) for_each_pipe(dev_priv, pipe) { seq_printf(m, "Pipe %c\n", pipe_name(pipe)); - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { entry = &ddb->plane[pipe][plane]; seq_printf(m, " Plane%-8d%8u%8u%8u\n", plane + 1, entry->start, entry->end, @@ -4009,8 +4050,7 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv, bool enable) { struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A); struct intel_crtc_state *pipe_config; struct drm_atomic_state *state; int ret = 0; @@ -4076,10 +4116,8 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, enum pipe pipe, enum intel_pipe_crc_source source) { - struct drm_device *dev = &dev_priv->drm; struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; - struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); enum intel_display_power_domain power_domain; u32 val = 0; /* shut up gcc */ int ret; @@ -4150,15 +4188,15 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, /* real source -> none transition */ if (source == INTEL_PIPE_CRC_SOURCE_NONE) { struct intel_pipe_crc_entry *entries; - struct intel_crtc *crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); DRM_DEBUG_DRIVER("stopping CRCs for pipe %c\n", pipe_name(pipe)); drm_modeset_lock(&crtc->base.mutex, NULL); if (crtc->base.state->active) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); drm_modeset_unlock(&crtc->base.mutex); spin_lock_irq(&pipe_crc->lock); @@ -4798,13 +4836,9 @@ i915_wedged_set(void *data, u64 val) if (i915_reset_in_progress(&dev_priv->gpu_error)) return -EAGAIN; - intel_runtime_pm_get(dev_priv); - i915_handle_error(dev_priv, val, "Manually setting wedged to %llu", val); - intel_runtime_pm_put(dev_priv); - return 0; } @@ -4872,10 +4906,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops, #define DROP_BOUND 0x2 #define DROP_RETIRE 0x4 #define DROP_ACTIVE 0x8 -#define DROP_ALL (DROP_UNBOUND | \ - DROP_BOUND | \ - DROP_RETIRE | \ - DROP_ACTIVE) +#define DROP_FREED 0x10 +#define DROP_ALL (DROP_UNBOUND | \ + DROP_BOUND | \ + DROP_RETIRE | \ + DROP_ACTIVE | \ + DROP_FREED) static int i915_drop_caches_get(void *data, u64 *val) { @@ -4919,6 +4955,11 @@ i915_drop_caches_set(void *data, u64 val) unlock: mutex_unlock(&dev->struct_mutex); + if (val & DROP_FREED) { + synchronize_rcu(); + flush_work(&dev_priv->mm.free_work); + } + return ret; } @@ -5039,22 +5080,16 @@ static int i915_cache_sharing_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - struct drm_device *dev = &dev_priv->drm; u32 snpcr; - int ret; if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv))) return -ENODEV; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); *val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT; @@ -5347,10 +5382,6 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_gem_seqno", i915_gem_seqno_info, 0}, {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0}, {"i915_gem_interrupt", i915_interrupt_info, 0}, - {"i915_gem_hws", i915_hws_info, 0, (void *)RCS}, - {"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS}, - {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS}, - {"i915_gem_hws_vebox", i915_hws_info, 0, (void *)VECS}, {"i915_gem_batch_pool", i915_gem_batch_pool_info, 0}, {"i915_guc_info", i915_guc_info, 0}, {"i915_guc_load_status", i915_guc_load_status_info, 0}, @@ -5414,7 +5445,8 @@ static const struct i915_debugfs_files { {"i915_fbc_false_color", &i915_fbc_fc_fops}, {"i915_dp_test_data", &i915_displayport_test_data_fops}, {"i915_dp_test_type", &i915_displayport_test_type_fops}, - {"i915_dp_test_active", &i915_displayport_test_active_fops} + {"i915_dp_test_active", &i915_displayport_test_active_fops}, + {"i915_guc_log_control", &i915_guc_log_control_fops} }; void intel_display_crc_init(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 912d5348e3e7..445fec9c2841 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -150,7 +150,7 @@ static void intel_detect_pch(struct drm_device *dev) /* In all current cases, num_pipes is equivalent to the PCH_NOP setting * (which really amounts to a PCH but no South Display). */ - if (INTEL_INFO(dev)->num_pipes == 0) { + if (INTEL_INFO(dev_priv)->num_pipes == 0) { dev_priv->pch_type = PCH_NOP; return; } @@ -323,6 +323,10 @@ static int i915_getparam(struct drm_device *dev, void *data, */ value = i915_gem_mmap_gtt_version(); break; + case I915_PARAM_HAS_SCHEDULER: + value = dev_priv->engine[RCS] && + dev_priv->engine[RCS]->schedule; + break; case I915_PARAM_MMAP_VERSION: /* Remember to bump this if the version changes! */ case I915_PARAM_HAS_GEM: @@ -374,12 +378,12 @@ static int intel_alloc_mchbar_resource(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; + int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; u32 temp_lo, temp_hi = 0; u64 mchbar_addr; int ret; - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); mchbar_addr = ((u64)temp_hi << 32) | temp_lo; @@ -406,7 +410,7 @@ intel_alloc_mchbar_resource(struct drm_device *dev) return ret; } - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) pci_write_config_dword(dev_priv->bridge_dev, reg + 4, upper_32_bits(dev_priv->mch_res.start)); @@ -420,7 +424,7 @@ static void intel_setup_mchbar(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; + int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; u32 temp; bool enabled; @@ -460,7 +464,7 @@ static void intel_teardown_mchbar(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; + int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; if (dev_priv->mchbar_need_disable) { if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { @@ -491,7 +495,7 @@ static unsigned int i915_vga_set_decode(void *cookie, bool state) { struct drm_device *dev = cookie; - intel_modeset_vga_set_state(dev, state); + intel_modeset_vga_set_state(to_i915(dev), state); if (state) return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; @@ -537,14 +541,17 @@ static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { .can_switch = i915_switcheroo_can_switch, }; -static void i915_gem_fini(struct drm_device *dev) +static void i915_gem_fini(struct drm_i915_private *dev_priv) { - mutex_lock(&dev->struct_mutex); - i915_gem_cleanup_engines(dev); - i915_gem_context_fini(dev); - mutex_unlock(&dev->struct_mutex); + mutex_lock(&dev_priv->drm.struct_mutex); + i915_gem_cleanup_engines(&dev_priv->drm); + i915_gem_context_fini(&dev_priv->drm); + mutex_unlock(&dev_priv->drm.struct_mutex); - WARN_ON(!list_empty(&to_i915(dev)->context_list)); + rcu_barrier(); + flush_work(&dev_priv->mm.free_work); + + WARN_ON(!list_empty(&dev_priv->context_list)); } static int i915_load_modeset_init(struct drm_device *dev) @@ -592,7 +599,9 @@ static int i915_load_modeset_init(struct drm_device *dev) /* Important: The output setup functions called by modeset_init need * working irqs for e.g. gmbus and dp aux transfers. */ - intel_modeset_init(dev); + ret = intel_modeset_init(dev); + if (ret) + goto cleanup_irq; intel_guc_init(dev); @@ -602,7 +611,7 @@ static int i915_load_modeset_init(struct drm_device *dev) intel_modeset_gem_init(dev); - if (INTEL_INFO(dev)->num_pipes == 0) + if (INTEL_INFO(dev_priv)->num_pipes == 0) return 0; ret = intel_fbdev_init(dev); @@ -619,7 +628,7 @@ static int i915_load_modeset_init(struct drm_device *dev) cleanup_gem: if (i915_gem_suspend(dev)) DRM_ERROR("failed to idle hardware; continuing to unload!\n"); - i915_gem_fini(dev); + i915_gem_fini(dev_priv); cleanup_irq: intel_guc_fini(dev); drm_irq_uninstall(dev); @@ -825,10 +834,13 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv, intel_init_dpio(dev_priv); intel_power_domains_init(dev_priv); intel_irq_init(dev_priv); + intel_hangcheck_init(dev_priv); intel_init_display_hooks(dev_priv); intel_init_clock_gating_hooks(dev_priv); intel_init_audio_hooks(dev_priv); - i915_gem_load_init(&dev_priv->drm); + ret = i915_gem_load_init(&dev_priv->drm); + if (ret < 0) + goto err_gvt; intel_display_crc_init(dev_priv); @@ -838,6 +850,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv, return 0; +err_gvt: + intel_gvt_cleanup(dev_priv); err_workqueues: i915_workqueues_cleanup(dev_priv); return ret; @@ -869,7 +883,7 @@ static int i915_mmio_setup(struct drm_device *dev) * the register BAR remains the same size for all the earlier * generations up to Ironlake. */ - if (INTEL_INFO(dev)->gen < 5) + if (INTEL_GEN(dev_priv) < 5) mmio_size = 512 * 1024; else mmio_size = 2 * 1024 * 1024; @@ -972,7 +986,6 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv) static int i915_driver_init_hw(struct drm_i915_private *dev_priv) { struct pci_dev *pdev = dev_priv->drm.pdev; - struct drm_device *dev = &dev_priv->drm; int ret; if (i915_inject_load_failure()) @@ -1030,7 +1043,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) * behaviour if any general state is accessed within a page above 4GB, * which also needs to be handled carefully. */ - if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) { + if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv)) { ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { @@ -1111,6 +1124,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) /* Reveal our presence to userspace */ if (drm_dev_register(dev, 0) == 0) { i915_debugfs_register(dev_priv); + i915_guc_register(dev_priv); i915_setup_sysfs(dev_priv); } else DRM_ERROR("Failed to register driver for userspace access!\n"); @@ -1149,6 +1163,7 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv) intel_opregion_unregister(dev_priv); i915_teardown_sysfs(dev_priv); + i915_guc_unregister(dev_priv); i915_debugfs_unregister(dev_priv); drm_dev_unregister(&dev_priv->drm); @@ -1157,8 +1172,8 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv) /** * i915_driver_load - setup chip and create an initial config - * @dev: DRM device - * @flags: startup flags + * @pdev: PCI device + * @ent: matching PCI ID entry * * The driver load routine has to do several things: * - drive output discovery via intel_modeset_init() @@ -1303,7 +1318,7 @@ void i915_driver_unload(struct drm_device *dev) drain_workqueue(dev_priv->wq); intel_guc_fini(dev); - i915_gem_fini(dev); + i915_gem_fini(dev_priv); intel_fbc_cleanup_cfb(dev_priv); intel_power_domains_fini(dev_priv); @@ -1425,9 +1440,9 @@ static int i915_drm_suspend(struct drm_device *dev) intel_suspend_encoders(dev_priv); - intel_suspend_hw(dev); + intel_suspend_hw(dev_priv); - i915_gem_suspend_gtt_mappings(dev); + i915_gem_suspend_gtt_mappings(dev_priv); i915_save_state(dev); @@ -1501,7 +1516,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) * Fujitsu FSC S7110 * Acer Aspire 1830T */ - if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6)) + if (!(hibernation && INTEL_GEN(dev_priv) < 6)) pci_set_power_state(pdev, PCI_D3hot); dev_priv->suspended_to_idle = suspend_to_idle(dev_priv); @@ -1589,6 +1604,8 @@ static int i915_drm_resume(struct drm_device *dev) intel_display_resume(dev); + drm_kms_helper_poll_enable(dev); + /* * ... but also need to make sure that hotplug processing * doesn't cause havoc. Like in the driver load code we don't @@ -1596,8 +1613,6 @@ static int i915_drm_resume(struct drm_device *dev) * notifications. * */ intel_hpd_init(dev_priv); - /* Config may have changed between suspend and resume */ - drm_helper_hpd_irq_event(dev); intel_opregion_register(dev_priv); @@ -1610,7 +1625,6 @@ static int i915_drm_resume(struct drm_device *dev) intel_opregion_notify_adapter(dev_priv, PCI_D0); intel_autoenable_gt_powersave(dev_priv); - drm_kms_helper_poll_enable(dev); enable_rpm_wakeref_asserts(dev_priv); @@ -2254,7 +2268,6 @@ err1: static int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume) { - struct drm_device *dev = &dev_priv->drm; int err; int ret; @@ -2278,10 +2291,8 @@ static int vlv_resume_prepare(struct drm_i915_private *dev_priv, vlv_check_no_gt_access(dev_priv); - if (rpm_resume) { - intel_init_clock_gating(dev); - i915_gem_restore_fences(dev); - } + if (rpm_resume) + intel_init_clock_gating(dev_priv); return ret; } @@ -2301,32 +2312,13 @@ static int intel_runtime_suspend(struct device *kdev) DRM_DEBUG_KMS("Suspending device\n"); - /* - * We could deadlock here in case another thread holding struct_mutex - * calls RPM suspend concurrently, since the RPM suspend will wait - * first for this RPM suspend to finish. In this case the concurrent - * RPM resume will be followed by its RPM suspend counterpart. Still - * for consistency return -EAGAIN, which will reschedule this suspend. - */ - if (!mutex_trylock(&dev->struct_mutex)) { - DRM_DEBUG_KMS("device lock contention, deffering suspend\n"); - /* - * Bump the expiration timestamp, otherwise the suspend won't - * be rescheduled. - */ - pm_runtime_mark_last_busy(kdev); - - return -EAGAIN; - } - disable_rpm_wakeref_asserts(dev_priv); /* * We are safe here against re-faults, since the fault handler takes * an RPM reference. */ - i915_gem_release_all_mmaps(dev_priv); - mutex_unlock(&dev->struct_mutex); + i915_gem_runtime_suspend(dev_priv); intel_guc_suspend(dev); @@ -2434,7 +2426,7 @@ static int intel_runtime_resume(struct device *kdev) * No point of rolling back things in case of an error, as the best * we can do is to hope that things will still work (and disable RPM). */ - i915_gem_init_swizzling(dev); + i915_gem_init_swizzling(dev_priv); intel_runtime_pm_enable_interrupts(dev_priv); @@ -2509,9 +2501,7 @@ static const struct file_operations i915_driver_fops = { .mmap = drm_gem_mmap, .poll = drm_poll, .read = drm_read, -#ifdef CONFIG_COMPAT .compat_ioctl = i915_compat_ioctl, -#endif .llseek = noop_llseek, }; @@ -2591,7 +2581,7 @@ static struct drm_driver driver = { .set_busid = drm_pci_set_busid, .gem_close_object = i915_gem_close_object, - .gem_free_object = i915_gem_free_object, + .gem_free_object_unlocked = i915_gem_free_object, .gem_vm_ops = &i915_gem_vm_ops, .prime_handle_to_fd = drm_gem_prime_handle_to_fd, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index f022f438e5b9..56002a52936d 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -41,6 +41,7 @@ #include <linux/intel-iommu.h> #include <linux/kref.h> #include <linux/pm_qos.h> +#include <linux/reservation.h> #include <linux/shmem_fs.h> #include <drm/drmP.h> @@ -59,9 +60,14 @@ #include "intel_ringbuffer.h" #include "i915_gem.h" +#include "i915_gem_fence_reg.h" +#include "i915_gem_object.h" #include "i915_gem_gtt.h" #include "i915_gem_render_state.h" #include "i915_gem_request.h" +#include "i915_gem_timeline.h" + +#include "i915_vma.h" #include "intel_gvt.h" @@ -70,8 +76,8 @@ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20161024" -#define DRIVER_TIMESTAMP 1477290335 +#define DRIVER_DATE "20161121" +#define DRIVER_TIMESTAMP 1479717903 #undef WARN_ON /* Many gcc seem to no see through this and fall over :( */ @@ -123,6 +129,11 @@ static inline const char *onoff(bool v) return v ? "on" : "off"; } +static inline const char *enableddisabled(bool v) +{ + return v ? "enabled" : "disabled"; +} + enum pipe { INVALID_PIPE = -1, PIPE_A = 0, @@ -183,7 +194,7 @@ enum plane { }; #define plane_name(p) ((p) + 'A') -#define sprite_name(p, s) ((p) * INTEL_INFO(dev)->num_sprites[(p)] + (s) + 'A') +#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') enum port { PORT_NONE = -1, @@ -312,7 +323,7 @@ struct i915_hotplug { #define for_each_pipe_masked(__dev_priv, __p, __mask) \ for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \ for_each_if ((__mask) & (1 << (__p))) -#define for_each_plane(__dev_priv, __pipe, __p) \ +#define for_each_universal_plane(__dev_priv, __pipe, __p) \ for ((__p) = 0; \ (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ (__p)++) @@ -457,23 +468,6 @@ struct intel_opregion { struct intel_overlay; struct intel_overlay_error_state; -struct drm_i915_fence_reg { - struct list_head link; - struct drm_i915_private *i915; - struct i915_vma *vma; - int pin_count; - int id; - /** - * Whether the tiling parameters for the currently - * associated fence register have changed. Note that - * for the purposes of tracking tiling changes we also - * treat the unfenced register, the register slot that - * the object occupies whilst it executes a fenced - * command (such as BLT on gen2/3), as a "fence". - */ - bool dirty; -}; - struct sdvo_device_mapping { u8 initialized; u8 dvo_port; @@ -485,6 +479,7 @@ struct sdvo_device_mapping { struct intel_connector; struct intel_encoder; +struct intel_atomic_state; struct intel_crtc_state; struct intel_initial_plane_config; struct intel_crtc; @@ -492,16 +487,20 @@ struct intel_limit; struct dpll; struct drm_i915_display_funcs { - int (*get_display_clock_speed)(struct drm_device *dev); - int (*get_fifo_size)(struct drm_device *dev, int plane); + int (*get_display_clock_speed)(struct drm_i915_private *dev_priv); + int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane); int (*compute_pipe_wm)(struct intel_crtc_state *cstate); int (*compute_intermediate_wm)(struct drm_device *dev, struct intel_crtc *intel_crtc, struct intel_crtc_state *newstate); - void (*initial_watermarks)(struct intel_crtc_state *cstate); - void (*optimize_watermarks)(struct intel_crtc_state *cstate); + void (*initial_watermarks)(struct intel_atomic_state *state, + struct intel_crtc_state *cstate); + void (*atomic_update_watermarks)(struct intel_atomic_state *state, + struct intel_crtc_state *cstate); + void (*optimize_watermarks)(struct intel_atomic_state *state, + struct intel_crtc_state *cstate); int (*compute_global_watermarks)(struct drm_atomic_state *state); - void (*update_wm)(struct drm_crtc *crtc); + void (*update_wm)(struct intel_crtc *crtc); int (*modeset_calc_cdclk)(struct drm_atomic_state *state); void (*modeset_commit_cdclk)(struct drm_atomic_state *state); /* Returns the active state of the crtc, and if the crtc is active, @@ -523,7 +522,7 @@ struct drm_i915_display_funcs { const struct drm_display_mode *adjusted_mode); void (*audio_codec_disable)(struct intel_encoder *encoder); void (*fdi_link_train)(struct drm_crtc *crtc); - void (*init_clock_gating)(struct drm_device *dev); + void (*init_clock_gating)(struct drm_i915_private *dev_priv); int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, @@ -560,6 +559,18 @@ enum forcewake_domains { #define FW_REG_READ (1) #define FW_REG_WRITE (2) +enum decoupled_power_domain { + GEN9_DECOUPLED_PD_BLITTER = 0, + GEN9_DECOUPLED_PD_RENDER, + GEN9_DECOUPLED_PD_MEDIA, + GEN9_DECOUPLED_PD_ALL +}; + +enum decoupled_ops { + GEN9_DECOUPLED_OP_WRITE = 0, + GEN9_DECOUPLED_OP_READ +}; + enum forcewake_domains intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv, i915_reg_t reg, unsigned int op); @@ -666,8 +677,9 @@ struct intel_csr { func(is_skylake); \ func(is_broxton); \ func(is_kabylake); \ - func(is_preliminary); \ + func(is_alpha_support); \ /* Keep has_* in alphabetical order */ \ + func(has_64bit_reloc); \ func(has_csr); \ func(has_ddi); \ func(has_dp_mst); \ @@ -693,7 +705,8 @@ struct intel_csr { func(cursor_needs_physical); \ func(hws_needs_physical); \ func(overlay_needs_physical); \ - func(supports_tv) + func(supports_tv); \ + func(has_decoupled_mmio) struct sseu_dev_info { u8 slice_mask; @@ -746,6 +759,8 @@ struct intel_display_error_state; struct drm_i915_error_state { struct kref ref; struct timeval time; + struct timeval boottime; + struct timeval uptime; struct drm_i915_private *i915; @@ -778,6 +793,7 @@ struct drm_i915_error_state { struct intel_overlay_error_state *overlay; struct intel_display_error_state *display; struct drm_i915_error_object *semaphore; + struct drm_i915_error_object *guc_log; struct drm_i915_error_engine { int engine_id; @@ -797,7 +813,6 @@ struct drm_i915_error_state { u32 cpu_ring_tail; u32 last_seqno; - u32 semaphore_seqno[I915_NUM_ENGINES - 1]; /* Register state */ u32 start; @@ -933,6 +948,7 @@ struct i915_gem_context { struct drm_i915_file_private *file_priv; struct i915_hw_ppgtt *ppgtt; struct pid *pid; + const char *name; struct i915_ctx_hang_stats hang_stats; @@ -943,6 +959,7 @@ struct i915_gem_context { /* Unique identifier for this context, used by the hw for tracking */ unsigned int hw_id; u32 user_handle; + int priority; /* greater priorities are serviced first */ u32 ggtt_alignment; @@ -1319,6 +1336,12 @@ struct i915_power_well { /* cached hw enabled state */ bool hw_enabled; unsigned long domains; + /* unique identifier for this power well */ + unsigned long id; + /* + * Arbitraty data associated with this power well. Platform and power + * well specific. + */ unsigned long data; const struct i915_power_well_ops *ops; }; @@ -1356,11 +1379,22 @@ struct i915_gem_mm { struct list_head bound_list; /** * List of objects which are not bound to the GTT (thus - * are idle and not used by the GPU) but still have - * (presumably uncached) pages still attached. + * are idle and not used by the GPU). These objects may or may + * not actually have any pages attached. */ struct list_head unbound_list; + /** List of all objects in gtt_space, currently mmaped by userspace. + * All objects within this list must also be on bound_list. + */ + struct list_head userfault_list; + + /** + * List of objects which are pending destruction. + */ + struct llist_head free_list; + struct work_struct free_work; + /** Usable portion of the GTT for GEM */ unsigned long stolen_base; /* limited to low memory (32-bit) */ @@ -1409,6 +1443,9 @@ struct i915_error_state_file_priv { struct drm_i915_error_state *error; }; +#define I915_RESET_TIMEOUT (10 * HZ) /* 10s */ +#define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */ + struct i915_gpu_error { /* For hangcheck timer */ #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ @@ -1682,7 +1719,6 @@ struct skl_wm_level { */ struct i915_runtime_pm { atomic_t wakeref_count; - atomic_t atomic_seq; bool suspended; bool irqs_enabled; }; @@ -1766,6 +1802,7 @@ struct drm_i915_private { struct kmem_cache *objects; struct kmem_cache *vmas; struct kmem_cache *requests; + struct kmem_cache *dependencies; const struct intel_device_info info; @@ -1807,7 +1844,6 @@ struct drm_i915_private { struct i915_gem_context *kernel_context; struct intel_engine_cs *engine[I915_NUM_ENGINES]; struct i915_vma *semaphore; - u32 next_seqno; struct drm_dma_handle *status_page_dmah; struct resource mch_res; @@ -1832,8 +1868,10 @@ struct drm_i915_private { u32 de_irq_mask[I915_MAX_PIPES]; }; u32 gt_irq_mask; - u32 pm_irq_mask; + u32 pm_imr; + u32 pm_ier; u32 pm_rps_events; + u32 pm_guc_events; u32 pipestat_irq_mask[I915_MAX_PIPES]; struct i915_hotplug hotplug; @@ -1910,8 +1948,8 @@ struct drm_i915_private { /* Kernel Modesetting */ - struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; - struct drm_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; wait_queue_head_t pending_flip_queue; #ifdef CONFIG_DEBUG_FS @@ -2027,13 +2065,6 @@ struct drm_i915_private { */ uint16_t skl_latency[8]; - /* - * The skl_wm_values structure is a bit too big for stack - * allocation, so we keep the staging struct where we store - * intermediate results here instead. - */ - struct skl_wm_values skl_results; - /* current hardware state */ union { struct ilk_wm_values hw; @@ -2065,6 +2096,10 @@ struct drm_i915_private { void (*resume)(struct drm_i915_private *); void (*cleanup_engine)(struct intel_engine_cs *engine); + struct list_head timelines; + struct i915_gem_timeline global_timeline; + u32 active_requests; + /** * Is the GPU currently considered idle, or busy executing * userspace requests? Whilst idle, we allow runtime power @@ -2072,7 +2107,6 @@ struct drm_i915_private { * In order to reduce the effect on performance, there * is a slight delay before we do so. */ - unsigned int active_engines; bool awake; /** @@ -2092,6 +2126,8 @@ struct drm_i915_private { * off the idle_work. */ struct delayed_work idle_work; + + ktime_t last_init_time; } gt; /* perform PHY state sanity checks? */ @@ -2148,30 +2184,6 @@ enum hdmi_force_audio { #define I915_GTT_OFFSET_NONE ((u32)-1) -struct drm_i915_gem_object_ops { - unsigned int flags; -#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1 - - /* Interface between the GEM object and its backing storage. - * get_pages() is called once prior to the use of the associated set - * of pages before to binding them into the GTT, and put_pages() is - * called after we no longer need them. As we expect there to be - * associated cost with migrating pages between the backing storage - * and making them available for the GPU (e.g. clflush), we may hold - * onto the pages after they are no longer referenced by the GPU - * in case they may be used again shortly (for example migrating the - * pages to a different memory domain within the GTT). put_pages() - * will therefore most likely be called when the object itself is - * being released or under memory pressure (where we attempt to - * reap pages for the shrinker). - */ - int (*get_pages)(struct drm_i915_gem_object *); - void (*put_pages)(struct drm_i915_gem_object *); - - int (*dmabuf_export)(struct drm_i915_gem_object *); - void (*release)(struct drm_i915_gem_object *); -}; - /* * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is * considered to be the frontbuffer for the given plane interface-wise. This @@ -2193,232 +2205,6 @@ struct drm_i915_gem_object_ops { #define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) -struct drm_i915_gem_object { - struct drm_gem_object base; - - const struct drm_i915_gem_object_ops *ops; - - /** List of VMAs backed by this object */ - struct list_head vma_list; - - /** Stolen memory for this object, instead of being backed by shmem. */ - struct drm_mm_node *stolen; - struct list_head global_list; - - /** Used in execbuf to temporarily hold a ref */ - struct list_head obj_exec_link; - - struct list_head batch_pool_link; - - unsigned long flags; - /** - * This is set if the object is on the active lists (has pending - * rendering and so a non-zero seqno), and is not set if it i s on - * inactive (ready to be unbound) list. - */ -#define I915_BO_ACTIVE_SHIFT 0 -#define I915_BO_ACTIVE_MASK ((1 << I915_NUM_ENGINES) - 1) -#define __I915_BO_ACTIVE(bo) \ - ((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK) - - /** - * This is set if the object has been written to since last bound - * to the GTT - */ - unsigned int dirty:1; - - /** - * Advice: are the backing pages purgeable? - */ - unsigned int madv:2; - - /** - * Whether the current gtt mapping needs to be mappable (and isn't just - * mappable by accident). Track pin and fault separate for a more - * accurate mappable working set. - */ - unsigned int fault_mappable:1; - - /* - * Is the object to be mapped as read-only to the GPU - * Only honoured if hardware has relevant pte bit - */ - unsigned long gt_ro:1; - unsigned int cache_level:3; - unsigned int cache_dirty:1; - - atomic_t frontbuffer_bits; - unsigned int frontbuffer_ggtt_origin; /* write once */ - - /** Current tiling stride for the object, if it's tiled. */ - unsigned int tiling_and_stride; -#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */ -#define TILING_MASK (FENCE_MINIMUM_STRIDE-1) -#define STRIDE_MASK (~TILING_MASK) - - /** Count of VMA actually bound by this object */ - unsigned int bind_count; - unsigned int pin_display; - - struct sg_table *pages; - int pages_pin_count; - struct get_page { - struct scatterlist *sg; - int last; - } get_page; - void *mapping; - - /** Breadcrumb of last rendering to the buffer. - * There can only be one writer, but we allow for multiple readers. - * If there is a writer that necessarily implies that all other - * read requests are complete - but we may only be lazily clearing - * the read requests. A read request is naturally the most recent - * request on a ring, so we may have two different write and read - * requests on one ring where the write request is older than the - * read request. This allows for the CPU to read from an active - * buffer by only waiting for the write to complete. - */ - struct i915_gem_active last_read[I915_NUM_ENGINES]; - struct i915_gem_active last_write; - - /** References from framebuffers, locks out tiling changes. */ - unsigned long framebuffer_references; - - /** Record of address bit 17 of each page at last unbind. */ - unsigned long *bit_17; - - struct i915_gem_userptr { - uintptr_t ptr; - unsigned read_only :1; - unsigned workers :4; -#define I915_GEM_USERPTR_MAX_WORKERS 15 - - struct i915_mm_struct *mm; - struct i915_mmu_object *mmu_object; - struct work_struct *work; - } userptr; - - /** for phys allocated objects */ - struct drm_dma_handle *phys_handle; -}; - -static inline struct drm_i915_gem_object * -to_intel_bo(struct drm_gem_object *gem) -{ - /* Assert that to_intel_bo(NULL) == NULL */ - BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base)); - - return container_of(gem, struct drm_i915_gem_object, base); -} - -static inline struct drm_i915_gem_object * -i915_gem_object_lookup(struct drm_file *file, u32 handle) -{ - return to_intel_bo(drm_gem_object_lookup(file, handle)); -} - -__deprecated -extern struct drm_gem_object * -drm_gem_object_lookup(struct drm_file *file, u32 handle); - -__attribute__((nonnull)) -static inline struct drm_i915_gem_object * -i915_gem_object_get(struct drm_i915_gem_object *obj) -{ - drm_gem_object_reference(&obj->base); - return obj; -} - -__deprecated -extern void drm_gem_object_reference(struct drm_gem_object *); - -__attribute__((nonnull)) -static inline void -i915_gem_object_put(struct drm_i915_gem_object *obj) -{ - drm_gem_object_unreference(&obj->base); -} - -__deprecated -extern void drm_gem_object_unreference(struct drm_gem_object *); - -__attribute__((nonnull)) -static inline void -i915_gem_object_put_unlocked(struct drm_i915_gem_object *obj) -{ - drm_gem_object_unreference_unlocked(&obj->base); -} - -__deprecated -extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *); - -static inline bool -i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) -{ - return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; -} - -static inline unsigned long -i915_gem_object_get_active(const struct drm_i915_gem_object *obj) -{ - return (obj->flags >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK; -} - -static inline bool -i915_gem_object_is_active(const struct drm_i915_gem_object *obj) -{ - return i915_gem_object_get_active(obj); -} - -static inline void -i915_gem_object_set_active(struct drm_i915_gem_object *obj, int engine) -{ - obj->flags |= BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline void -i915_gem_object_clear_active(struct drm_i915_gem_object *obj, int engine) -{ - obj->flags &= ~BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline bool -i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj, - int engine) -{ - return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline unsigned int -i915_gem_object_get_tiling(struct drm_i915_gem_object *obj) -{ - return obj->tiling_and_stride & TILING_MASK; -} - -static inline bool -i915_gem_object_is_tiled(struct drm_i915_gem_object *obj) -{ - return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; -} - -static inline unsigned int -i915_gem_object_get_stride(struct drm_i915_gem_object *obj) -{ - return obj->tiling_and_stride & STRIDE_MASK; -} - -static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) -{ - i915_gem_object_get(vma->obj); - return vma; -} - -static inline void i915_vma_put(struct i915_vma *vma) -{ - lockdep_assert_held(&vma->vm->dev->struct_mutex); - i915_gem_object_put(vma->obj); -} - /* * Optimised SGL iterator for GEM objects */ @@ -2445,6 +2231,14 @@ static __always_inline struct sgt_iter { return s; } +static inline struct scatterlist *____sg_next(struct scatterlist *sg) +{ + ++sg; + if (unlikely(sg_is_chain(sg))) + sg = sg_chain_ptr(sg); + return sg; +} + /** * __sg_next - return the next scatterlist entry in a list * @sg: The current sg entry @@ -2459,9 +2253,7 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg) #ifdef CONFIG_DEBUG_SG BUG_ON(sg->sg_magic != SG_MAGIC); #endif - return sg_is_last(sg) ? NULL : - likely(!sg_is_chain(++sg)) ? sg : - sg_chain_ptr(sg); + return sg_is_last(sg) ? NULL : ____sg_next(sg); } /** @@ -2585,24 +2377,19 @@ struct drm_i915_cmd_table { int count; }; -/* Note that the (struct drm_i915_private *) cast is just to shut up gcc. */ -#define __I915__(p) ({ \ - struct drm_i915_private *__p; \ - if (__builtin_types_compatible_p(typeof(*p), struct drm_i915_private)) \ - __p = (struct drm_i915_private *)p; \ - else if (__builtin_types_compatible_p(typeof(*p), struct drm_device)) \ - __p = to_i915((struct drm_device *)p); \ - else \ - BUILD_BUG(); \ - __p; \ -}) -#define INTEL_INFO(p) (&__I915__(p)->info) +static inline const struct intel_device_info * +intel_info(const struct drm_i915_private *dev_priv) +{ + return &dev_priv->info; +} + +#define INTEL_INFO(dev_priv) intel_info((dev_priv)) #define INTEL_GEN(dev_priv) ((dev_priv)->info.gen) #define INTEL_DEVID(dev_priv) ((dev_priv)->info.device_id) #define REVID_FOREVER 0xff -#define INTEL_REVID(p) (__I915__(p)->drm.pdev->revision) +#define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision) #define GEN_FOREVER (0) /* @@ -2633,20 +2420,20 @@ struct drm_i915_cmd_table { #define IS_I830(dev_priv) (INTEL_DEVID(dev_priv) == 0x3577) #define IS_845G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2562) -#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) +#define IS_I85X(dev_priv) ((dev_priv)->info.is_i85x) #define IS_I865G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2572) -#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) +#define IS_I915G(dev_priv) ((dev_priv)->info.is_i915g) #define IS_I915GM(dev_priv) (INTEL_DEVID(dev_priv) == 0x2592) #define IS_I945G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2772) -#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) -#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) -#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) +#define IS_I945GM(dev_priv) ((dev_priv)->info.is_i945gm) +#define IS_BROADWATER(dev_priv) ((dev_priv)->info.is_broadwater) +#define IS_CRESTLINE(dev_priv) ((dev_priv)->info.is_crestline) #define IS_GM45(dev_priv) (INTEL_DEVID(dev_priv) == 0x2A42) #define IS_G4X(dev_priv) ((dev_priv)->info.is_g4x) #define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001) #define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011) -#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) -#define IS_G33(dev) (INTEL_INFO(dev)->is_g33) +#define IS_PINEVIEW(dev_priv) ((dev_priv)->info.is_pineview) +#define IS_G33(dev_priv) ((dev_priv)->info.is_g33) #define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046) #define IS_IVYBRIDGE(dev_priv) ((dev_priv)->info.is_ivybridge) #define IS_IVB_GT1(dev_priv) (INTEL_DEVID(dev_priv) == 0x0156 || \ @@ -2659,7 +2446,7 @@ struct drm_i915_cmd_table { #define IS_SKYLAKE(dev_priv) ((dev_priv)->info.is_skylake) #define IS_BROXTON(dev_priv) ((dev_priv)->info.is_broxton) #define IS_KABYLAKE(dev_priv) ((dev_priv)->info.is_kabylake) -#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) +#define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile) #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) #define IS_BDW_ULT(dev_priv) (IS_BROADWELL(dev_priv) && \ @@ -2699,7 +2486,7 @@ struct drm_i915_cmd_table { #define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030) -#define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary) +#define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support) #define SKL_REVID_A0 0x0 #define SKL_REVID_B0 0x1 @@ -2753,28 +2540,31 @@ struct drm_i915_cmd_table { #define ALL_ENGINES (~0) #define HAS_ENGINE(dev_priv, id) \ - (!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id))) + (!!((dev_priv)->info.ring_mask & ENGINE_MASK(id))) #define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS) #define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2) #define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS) #define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS) -#define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) -#define HAS_SNOOP(dev) (INTEL_INFO(dev)->has_snoop) -#define HAS_EDRAM(dev) (!!(__I915__(dev)->edram_cap & EDRAM_ENABLED)) +#define HAS_LLC(dev_priv) ((dev_priv)->info.has_llc) +#define HAS_SNOOP(dev_priv) ((dev_priv)->info.has_snoop) +#define HAS_EDRAM(dev_priv) (!!((dev_priv)->edram_cap & EDRAM_ENABLED)) #define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) -#define HWS_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->hws_needs_physical) -#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->has_hw_contexts) -#define HAS_LOGICAL_RING_CONTEXTS(dev) (INTEL_INFO(dev)->has_logical_ring_contexts) -#define USES_PPGTT(dev) (i915.enable_ppgtt) -#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt >= 2) -#define USES_FULL_48BIT_PPGTT(dev) (i915.enable_ppgtt == 3) +#define HWS_NEEDS_PHYSICAL(dev_priv) ((dev_priv)->info.hws_needs_physical) + +#define HAS_HW_CONTEXTS(dev_priv) ((dev_priv)->info.has_hw_contexts) +#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \ + ((dev_priv)->info.has_logical_ring_contexts) +#define USES_PPGTT(dev_priv) (i915.enable_ppgtt) +#define USES_FULL_PPGTT(dev_priv) (i915.enable_ppgtt >= 2) +#define USES_FULL_48BIT_PPGTT(dev_priv) (i915.enable_ppgtt == 3) -#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) -#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) +#define HAS_OVERLAY(dev_priv) ((dev_priv)->info.has_overlay) +#define OVERLAY_NEEDS_PHYSICAL(dev_priv) \ + ((dev_priv)->info.overlay_needs_physical) /* Early gen2 have a totally busted CS tlb and require pinned batches. */ #define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_845G(dev_priv)) @@ -2791,8 +2581,8 @@ struct drm_i915_cmd_table { * legacy irq no. is shared with another device. The kernel then disables that * interrupt source and so prevents the other device from working properly. */ -#define HAS_AUX_IRQ(dev) (INTEL_INFO(dev)->gen >= 5) -#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->has_gmbus_irq) +#define HAS_AUX_IRQ(dev_priv) ((dev_priv)->info.gen >= 5) +#define HAS_GMBUS_IRQ(dev_priv) ((dev_priv)->info.has_gmbus_irq) /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte * rows, which changed the alignment requirements and fence programming. @@ -2800,38 +2590,40 @@ struct drm_i915_cmd_table { #define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \ !(IS_I915G(dev_priv) || \ IS_I915GM(dev_priv))) -#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) -#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) +#define SUPPORTS_TV(dev_priv) ((dev_priv)->info.supports_tv) +#define I915_HAS_HOTPLUG(dev_priv) ((dev_priv)->info.has_hotplug) -#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2) -#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) -#define HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) +#define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) +#define HAS_PIPE_CXSR(dev_priv) ((dev_priv)->info.has_pipe_cxsr) +#define HAS_FBC(dev_priv) ((dev_priv)->info.has_fbc) #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) -#define HAS_DP_MST(dev) (INTEL_INFO(dev)->has_dp_mst) +#define HAS_DP_MST(dev_priv) ((dev_priv)->info.has_dp_mst) -#define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi) -#define HAS_FPGA_DBG_UNCLAIMED(dev) (INTEL_INFO(dev)->has_fpga_dbg) -#define HAS_PSR(dev) (INTEL_INFO(dev)->has_psr) -#define HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6) -#define HAS_RC6p(dev) (INTEL_INFO(dev)->has_rc6p) +#define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi) +#define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg) +#define HAS_PSR(dev_priv) ((dev_priv)->info.has_psr) +#define HAS_RC6(dev_priv) ((dev_priv)->info.has_rc6) +#define HAS_RC6p(dev_priv) ((dev_priv)->info.has_rc6p) -#define HAS_CSR(dev) (INTEL_INFO(dev)->has_csr) +#define HAS_CSR(dev_priv) ((dev_priv)->info.has_csr) #define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm) +#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc) + /* * For now, anything with a GuC requires uCode loading, and then supports * command submission once loaded. But these are logically independent * properties, so we have separate macros to test them. */ -#define HAS_GUC(dev) (INTEL_INFO(dev)->has_guc) -#define HAS_GUC_UCODE(dev) (HAS_GUC(dev)) -#define HAS_GUC_SCHED(dev) (HAS_GUC(dev)) +#define HAS_GUC(dev_priv) ((dev_priv)->info.has_guc) +#define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) +#define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv)) -#define HAS_RESOURCE_STREAMER(dev) (INTEL_INFO(dev)->has_resource_streamer) +#define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer) -#define HAS_POOLED_EU(dev) (INTEL_INFO(dev)->has_pooled_eu) +#define HAS_POOLED_EU(dev_priv) ((dev_priv)->info.has_pooled_eu) #define INTEL_PCH_DEVICE_ID_MASK 0xff00 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 @@ -2871,6 +2663,8 @@ struct drm_i915_cmd_table { #define GT_FREQUENCY_MULTIPLIER 50 #define GEN9_FREQ_SCALER 3 +#define HAS_DECOUPLED_MMIO(dev_priv) (INTEL_INFO(dev_priv)->has_decoupled_mmio) + #include "i915_trace.h" static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) @@ -2901,6 +2695,8 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level, #ifdef CONFIG_COMPAT extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +#else +#define i915_compat_ioctl NULL #endif extern const struct dev_pm_ops i915_pm_ops; @@ -2912,6 +2708,7 @@ extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv); extern void i915_reset(struct drm_i915_private *dev_priv); extern int intel_guc_reset(struct drm_i915_private *dev_priv); extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); +extern void intel_hangcheck_init(struct drm_i915_private *dev_priv); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); @@ -3095,7 +2892,7 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); -void i915_gem_load_init(struct drm_device *dev); +int i915_gem_load_init(struct drm_device *dev); void i915_gem_load_cleanup(struct drm_device *dev); void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); int i915_gem_freeze(struct drm_i915_private *dev_priv); @@ -3119,78 +2916,86 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, u64 alignment, u64 flags); -int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, - u32 flags); -void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); -int __must_check i915_vma_unbind(struct i915_vma *vma); -void i915_vma_close(struct i915_vma *vma); -void i915_vma_destroy(struct i915_vma *vma); - int i915_gem_object_unbind(struct drm_i915_gem_object *obj); -int i915_gem_object_put_pages(struct drm_i915_gem_object *obj); -void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); -int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj); +void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); -static inline int __sg_page_count(struct scatterlist *sg) +static inline int __sg_page_count(const struct scatterlist *sg) { return sg->length >> PAGE_SHIFT; } +struct scatterlist * +i915_gem_object_get_sg(struct drm_i915_gem_object *obj, + unsigned int n, unsigned int *offset); + +struct page * +i915_gem_object_get_page(struct drm_i915_gem_object *obj, + unsigned int n); + struct page * -i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n); +i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, + unsigned int n); -static inline dma_addr_t -i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n) +dma_addr_t +i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, + unsigned long n); + +void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages); +int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); + +static inline int __must_check +i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { - if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - } + might_lock(&obj->mm.lock); - while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) { - obj->get_page.last += __sg_page_count(obj->get_page.sg++); - if (unlikely(sg_is_chain(obj->get_page.sg))) - obj->get_page.sg = sg_chain_ptr(obj->get_page.sg); - } + if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) + return 0; - return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT); + return __i915_gem_object_get_pages(obj); } -static inline struct page * -i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n) +static inline void +__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { - if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT)) - return NULL; - - if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - } + GEM_BUG_ON(!obj->mm.pages); - while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) { - obj->get_page.last += __sg_page_count(obj->get_page.sg++); - if (unlikely(sg_is_chain(obj->get_page.sg))) - obj->get_page.sg = sg_chain_ptr(obj->get_page.sg); - } + atomic_inc(&obj->mm.pages_pin_count); +} - return nth_page(sg_page(obj->get_page.sg), n - obj->get_page.last); +static inline bool +i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) +{ + return atomic_read(&obj->mm.pages_pin_count); } -static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) +static inline void +__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { - GEM_BUG_ON(obj->pages == NULL); - obj->pages_pin_count++; + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + GEM_BUG_ON(!obj->mm.pages); + + atomic_dec(&obj->mm.pages_pin_count); + GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); } -static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) +static inline void +i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { - GEM_BUG_ON(obj->pages_pin_count == 0); - obj->pages_pin_count--; - GEM_BUG_ON(obj->pages_pin_count < obj->bind_count); + __i915_gem_object_unpin_pages(obj); } +enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock */ + I915_MM_NORMAL = 0, + I915_MM_SHRINKER +}; + +void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, + enum i915_mm_subclass subclass); +void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj); + enum i915_map_type { I915_MAP_WB = 0, I915_MAP_WC, @@ -3206,8 +3011,8 @@ enum i915_map_type { * the kernel address space. Based on the @type of mapping, the PTE will be * set to either WriteBack or WriteCombine (via pgprot_t). * - * The caller must hold the struct_mutex, and is responsible for calling - * i915_gem_object_unpin_map() when the mapping is no longer required. + * The caller is responsible for calling i915_gem_object_unpin_map() when the + * mapping is no longer required. * * Returns the pointer through which to access the mapped object, or an * ERR_PTR() on error. @@ -3223,12 +3028,9 @@ void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, * with your access, call i915_gem_object_unpin_map() to release the pin * upon the mapping. Once the pin count reaches zero, that mapping may be * removed. - * - * The caller must hold the struct_mutex. */ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) { - lockdep_assert_held(&obj->base.dev->struct_mutex); i915_gem_object_unpin_pages(obj); } @@ -3261,7 +3063,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, struct drm_i915_gem_object *new, unsigned frontbuffer_bits); -int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); +int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); struct drm_i915_gem_request * i915_gem_find_active_request(struct intel_engine_cs *engine); @@ -3290,19 +3092,25 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error) void i915_gem_reset(struct drm_i915_private *dev_priv); void i915_gem_set_wedged(struct drm_i915_private *dev_priv); -bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); +void i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); int __must_check i915_gem_init(struct drm_device *dev); int __must_check i915_gem_init_hw(struct drm_device *dev); -void i915_gem_init_swizzling(struct drm_device *dev); +void i915_gem_init_swizzling(struct drm_i915_private *dev_priv); void i915_gem_cleanup_engines(struct drm_device *dev); int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, unsigned int flags); int __must_check i915_gem_suspend(struct drm_device *dev); void i915_gem_resume(struct drm_device *dev); int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); -int __must_check -i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, - bool readonly); +int i915_gem_object_wait(struct drm_i915_gem_object *obj, + unsigned int flags, + long timeout, + struct intel_rps_client *rps); +int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, + unsigned int flags, + int priority); +#define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX + int __must_check i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); @@ -3362,57 +3170,17 @@ i915_gem_object_ggtt_offset(struct drm_i915_gem_object *o, return i915_ggtt_offset(i915_gem_object_to_ggtt(o, view)); } -/* i915_gem_fence.c */ +/* i915_gem_fence_reg.c */ int __must_check i915_vma_get_fence(struct i915_vma *vma); int __must_check i915_vma_put_fence(struct i915_vma *vma); -/** - * i915_vma_pin_fence - pin fencing state - * @vma: vma to pin fencing for - * - * This pins the fencing state (whether tiled or untiled) to make sure the - * vma (and its object) is ready to be used as a scanout target. Fencing - * status must be synchronize first by calling i915_vma_get_fence(): - * - * The resulting fence pin reference must be released again with - * i915_vma_unpin_fence(). - * - * Returns: - * - * True if the vma has a fence, false otherwise. - */ -static inline bool -i915_vma_pin_fence(struct i915_vma *vma) -{ - if (vma->fence) { - vma->fence->pin_count++; - return true; - } else - return false; -} +void i915_gem_restore_fences(struct drm_i915_private *dev_priv); -/** - * i915_vma_unpin_fence - unpin fencing state - * @vma: vma to unpin fencing for - * - * This releases the fence pin reference acquired through - * i915_vma_pin_fence. It will handle both objects with and without an - * attached fence correctly, callers do not need to distinguish this. - */ -static inline void -i915_vma_unpin_fence(struct i915_vma *vma) -{ - if (vma->fence) { - GEM_BUG_ON(vma->fence->pin_count <= 0); - vma->fence->pin_count--; - } -} - -void i915_gem_restore_fences(struct drm_device *dev); - -void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); -void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); -void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); +void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv); +void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages); +void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages); /* i915_gem_context.c */ int __must_check i915_gem_context_init(struct drm_device *dev); @@ -3422,6 +3190,9 @@ int i915_gem_context_open(struct drm_device *dev, struct drm_file *file); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); int i915_switch_context(struct drm_i915_gem_request *req); int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv); +struct i915_vma * +i915_gem_context_pin_legacy(struct i915_gem_context *ctx, + unsigned int flags); void i915_gem_context_free(struct kref *ctx_ref); struct drm_i915_gem_object * i915_gem_alloc_context_obj(struct drm_device *dev, size_t size); @@ -3455,6 +3226,16 @@ static inline void i915_gem_context_put(struct i915_gem_context *ctx) kref_put(&ctx->ref, i915_gem_context_free); } +static inline struct intel_timeline * +i915_gem_context_lookup_timeline(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) +{ + struct i915_address_space *vm; + + vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base; + return &vm->timeline.engine[engine->id]; +} + static inline bool i915_gem_context_is_default(const struct i915_gem_context *c) { return c->user_handle == DEFAULT_CONTEXT_HANDLE; @@ -3498,7 +3279,7 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, u64 end); void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, struct drm_mm_node *node); -int i915_gem_init_stolen(struct drm_device *dev); +int i915_gem_init_stolen(struct drm_i915_private *dev_priv); void i915_gem_cleanup_stolen(struct drm_device *dev); struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_device *dev, u32 size); @@ -3508,6 +3289,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal(struct drm_i915_private *dev_priv, + unsigned int size); + /* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target, @@ -3690,15 +3476,16 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv); /* modesetting */ extern void intel_modeset_init_hw(struct drm_device *dev); -extern void intel_modeset_init(struct drm_device *dev); +extern int intel_modeset_init(struct drm_device *dev); extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); extern int intel_connector_register(struct drm_connector *); extern void intel_connector_unregister(struct drm_connector *); -extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); +extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, + bool state); extern void intel_display_resume(struct drm_device *dev); -extern void i915_redisable_vga(struct drm_device *dev); -extern void i915_redisable_vga_power_on(struct drm_device *dev); +extern void i915_redisable_vga(struct drm_i915_private *dev_priv); +extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv); extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); extern void intel_init_pch_refclk(struct drm_device *dev); extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val); @@ -3717,7 +3504,7 @@ extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e, extern struct intel_display_error_state * intel_display_capture_error_state(struct drm_i915_private *dev_priv); extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, - struct drm_device *dev, + struct drm_i915_private *dev_priv, struct intel_display_error_state *error); int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); @@ -3745,6 +3532,23 @@ u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg); void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); /* intel_dpio_phy.c */ +void bxt_port_to_phy_channel(enum port port, + enum dpio_phy *phy, enum dpio_channel *ch); +void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, + enum port port, u32 margin, u32 scale, + u32 enable, u32 deemphasis); +void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); +void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); +bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, + enum dpio_phy phy); +bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, + enum dpio_phy phy); +uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, + uint8_t lane_count); +void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, + uint8_t lane_lat_optim_mask); +uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); + void chv_set_phy_signal_level(struct intel_encoder *encoder, u32 deemph_reg_value, u32 margin_reg_value, bool uniq_trans_scale); @@ -3834,11 +3638,30 @@ __raw_write(64, q) #undef __raw_write /* These are untraced mmio-accessors that are only valid to be used inside - * critical sections inside IRQ handlers where forcewake is explicitly + * critical sections, such as inside IRQ handlers, where forcewake is explicitly * controlled. + * * Think twice, and think again, before using these. - * Note: Should only be used between intel_uncore_forcewake_irqlock() and - * intel_uncore_forcewake_irqunlock(). + * + * As an example, these accessors can possibly be used between: + * + * spin_lock_irq(&dev_priv->uncore.lock); + * intel_uncore_forcewake_get__locked(); + * + * and + * + * intel_uncore_forcewake_put__locked(); + * spin_unlock_irq(&dev_priv->uncore.lock); + * + * + * Note: some registers may not need forcewake held, so + * intel_uncore_forcewake_{get,put} can be omitted, see + * intel_uncore_forcewake_for_reg(). + * + * Certain architectures will die if the same cacheline is concurrently accessed + * by different clients (e.g. on Ivybridge). Access to registers should + * therefore generally be serialised, by either the dev_priv->uncore.lock or + * a more localised lock guarding all access to that bank of registers. */ #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__)) #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__)) @@ -3915,7 +3738,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) /* Before we do the heavier coherent read of the seqno, * check the value (hopefully) in the CPU cacheline. */ - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; /* Ensure our read of the seqno is coherent so that we @@ -3966,7 +3789,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) wake_up_process(tsk); rcu_read_unlock(); - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; } diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index bf5e9064e02d..d0dcaf35b429 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -29,12 +29,12 @@ #include <drm/drm_vma_manager.h> #include <drm/i915_drm.h> #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "i915_vgpu.h" #include "i915_trace.h" #include "intel_drv.h" #include "intel_frontbuffer.h" #include "intel_mocs.h" +#include <linux/dma-fence-array.h> #include <linux/reservation.h> #include <linux/shmem_fs.h> #include <linux/slab.h> @@ -42,13 +42,14 @@ #include <linux/pci.h> #include <linux/dma-buf.h> +static void i915_gem_flush_free_objects(struct drm_i915_private *i915); static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); static bool cpu_cache_is_coherent(struct drm_device *dev, enum i915_cache_level level) { - return HAS_LLC(dev) || level != I915_CACHE_NONE; + return HAS_LLC(to_i915(dev)) || level != I915_CACHE_NONE; } static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) @@ -63,13 +64,13 @@ static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) } static int -insert_mappable_node(struct drm_i915_private *i915, +insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size) { memset(node, 0, sizeof(*node)); - return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node, - size, 0, 0, 0, - i915->ggtt.mappable_end, + return drm_mm_insert_node_in_range_generic(&ggtt->base.mm, node, + size, 0, -1, + 0, ggtt->mappable_end, DRM_MM_SEARCH_DEFAULT, DRM_MM_CREATE_DEFAULT); } @@ -104,6 +105,8 @@ i915_gem_wait_for_error(struct i915_gpu_error *error) { int ret; + might_sleep(); + if (!i915_reset_in_progress(error)) return 0; @@ -114,7 +117,7 @@ i915_gem_wait_for_error(struct i915_gpu_error *error) */ ret = wait_event_interruptible_timeout(error->reset_queue, !i915_reset_in_progress(error), - 10*HZ); + I915_RESET_TIMEOUT); if (ret == 0) { DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); return -EIO; @@ -167,7 +170,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, return 0; } -static int +static struct sg_table * i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) { struct address_space *mapping = obj->base.filp->f_mapping; @@ -177,7 +180,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) int i; if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) - return -EINVAL; + return ERR_PTR(-EINVAL); for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { struct page *page; @@ -185,7 +188,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) page = shmem_read_mapping_page(mapping, i); if (IS_ERR(page)) - return PTR_ERR(page); + return ERR_CAST(page); src = kmap_atomic(page); memcpy(vaddr, src, PAGE_SIZE); @@ -200,11 +203,11 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) st = kmalloc(sizeof(*st), GFP_KERNEL); if (st == NULL) - return -ENOMEM; + return ERR_PTR(-ENOMEM); if (sg_alloc_table(st, 1, GFP_KERNEL)) { kfree(st); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } sg = st->sgl; @@ -214,29 +217,33 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) sg_dma_address(sg) = obj->phys_handle->busaddr; sg_dma_len(sg) = obj->base.size; - obj->pages = st; - return 0; + return st; } static void -i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) +__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - int ret; + GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); - BUG_ON(obj->madv == __I915_MADV_PURGED); + if (obj->mm.madv == I915_MADV_DONTNEED) + obj->mm.dirty = false; - ret = i915_gem_object_set_to_cpu_domain(obj, true); - if (WARN_ON(ret)) { - /* In the event of a disaster, abandon all caches and - * hope for the best. - */ - obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; - } + if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 && + !cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) + drm_clflush_sg(pages); - if (obj->madv == I915_MADV_DONTNEED) - obj->dirty = 0; + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + obj->base.write_domain = I915_GEM_DOMAIN_CPU; +} - if (obj->dirty) { +static void +i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, + struct sg_table *pages) +{ + __i915_gem_object_release_shmem(obj, pages); + + if (obj->mm.dirty) { struct address_space *mapping = obj->base.filp->f_mapping; char *vaddr = obj->phys_handle->vaddr; int i; @@ -255,22 +262,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) kunmap_atomic(dst); set_page_dirty(page); - if (obj->madv == I915_MADV_WILLNEED) + if (obj->mm.madv == I915_MADV_WILLNEED) mark_page_accessed(page); put_page(page); vaddr += PAGE_SIZE; } - obj->dirty = 0; + obj->mm.dirty = false; } - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(pages); + kfree(pages); } static void i915_gem_object_release_phys(struct drm_i915_gem_object *obj) { drm_pci_free(obj->base.dev, obj->phys_handle); + i915_gem_object_unpin_pages(obj); } static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { @@ -292,7 +300,12 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj) * must wait for all rendering to complete to the object (as unbinding * must anyway), and retire the requests. */ - ret = i915_gem_object_wait_rendering(obj, false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; @@ -311,90 +324,209 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj) return ret; } -/** - * Ensures that all rendering to the object has completed and the object is - * safe to unbind from the GTT or access from the CPU. - * @obj: i915 gem object - * @readonly: waiting for just read access or read-write access - */ -int -i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, - bool readonly) +static long +i915_gem_object_wait_fence(struct dma_fence *fence, + unsigned int flags, + long timeout, + struct intel_rps_client *rps) { - struct reservation_object *resv; - struct i915_gem_active *active; - unsigned long active_mask; - int idx; + struct drm_i915_gem_request *rq; - lockdep_assert_held(&obj->base.dev->struct_mutex); + BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); - if (!readonly) { - active = obj->last_read; - active_mask = i915_gem_object_get_active(obj); - } else { - active_mask = 1; - active = &obj->last_write; + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return timeout; + + if (!dma_fence_is_i915(fence)) + return dma_fence_wait_timeout(fence, + flags & I915_WAIT_INTERRUPTIBLE, + timeout); + + rq = to_request(fence); + if (i915_gem_request_completed(rq)) + goto out; + + /* This client is about to stall waiting for the GPU. In many cases + * this is undesirable and limits the throughput of the system, as + * many clients cannot continue processing user input/output whilst + * blocked. RPS autotuning may take tens of milliseconds to respond + * to the GPU load and thus incurs additional latency for the client. + * We can circumvent that by promoting the GPU frequency to maximum + * before we wait. This makes the GPU throttle up much more quickly + * (good for benchmarks and user experience, e.g. window animations), + * but at a cost of spending more power processing the workload + * (bad for battery). Not all clients even want their results + * immediately and for them we should just let the GPU select its own + * frequency to maximise efficiency. To prevent a single client from + * forcing the clocks too high for the whole system, we only allow + * each client to waitboost once in a busy period. + */ + if (rps) { + if (INTEL_GEN(rq->i915) >= 6) + gen6_rps_boost(rq->i915, rps, rq->emitted_jiffies); + else + rps = NULL; + } + + timeout = i915_wait_request(rq, flags, timeout); + +out: + if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq)) + i915_gem_request_retire_upto(rq); + + if (rps && rq->global_seqno == intel_engine_last_submit(rq->engine)) { + /* The GPU is now idle and this client has stalled. + * Since no other client has submitted a request in the + * meantime, assume that this client is the only one + * supplying work to the GPU but is unable to keep that + * work supplied because it is waiting. Since the GPU is + * then never kept fully busy, RPS autoclocking will + * keep the clocks relatively low, causing further delays. + * Compensate by giving the synchronous client credit for + * a waitboost next time. + */ + spin_lock(&rq->i915->rps.client_lock); + list_del_init(&rps->link); + spin_unlock(&rq->i915->rps.client_lock); } - for_each_active(active_mask, idx) { + return timeout; +} + +static long +i915_gem_object_wait_reservation(struct reservation_object *resv, + unsigned int flags, + long timeout, + struct intel_rps_client *rps) +{ + struct dma_fence *excl; + + if (flags & I915_WAIT_ALL) { + struct dma_fence **shared; + unsigned int count, i; int ret; - ret = i915_gem_active_wait(&active[idx], - &obj->base.dev->struct_mutex); + ret = reservation_object_get_fences_rcu(resv, + &excl, &count, &shared); if (ret) return ret; - } - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - long err; + for (i = 0; i < count; i++) { + timeout = i915_gem_object_wait_fence(shared[i], + flags, timeout, + rps); + if (timeout <= 0) + break; - err = reservation_object_wait_timeout_rcu(resv, !readonly, true, - MAX_SCHEDULE_TIMEOUT); - if (err < 0) - return err; + dma_fence_put(shared[i]); + } + + for (; i < count; i++) + dma_fence_put(shared[i]); + kfree(shared); + } else { + excl = reservation_object_get_excl_rcu(resv); } - return 0; + if (excl && timeout > 0) + timeout = i915_gem_object_wait_fence(excl, flags, timeout, rps); + + dma_fence_put(excl); + + return timeout; } -/* A nonblocking variant of the above wait. Must be called prior to - * acquiring the mutex for the object, as the object state may change - * during this call. A reference must be held by the caller for the object. - */ -static __must_check int -__unsafe_wait_rendering(struct drm_i915_gem_object *obj, - struct intel_rps_client *rps, - bool readonly) +static void __fence_set_priority(struct dma_fence *fence, int prio) { - struct i915_gem_active *active; - unsigned long active_mask; - int idx; + struct drm_i915_gem_request *rq; + struct intel_engine_cs *engine; - active_mask = __I915_BO_ACTIVE(obj); - if (!active_mask) - return 0; + if (!dma_fence_is_i915(fence)) + return; + + rq = to_request(fence); + engine = rq->engine; + if (!engine->schedule) + return; + + engine->schedule(rq, prio); +} + +static void fence_set_priority(struct dma_fence *fence, int prio) +{ + /* Recurse once into a fence-array */ + if (dma_fence_is_array(fence)) { + struct dma_fence_array *array = to_dma_fence_array(fence); + int i; - if (!readonly) { - active = obj->last_read; + for (i = 0; i < array->num_fences; i++) + __fence_set_priority(array->fences[i], prio); } else { - active_mask = 1; - active = &obj->last_write; + __fence_set_priority(fence, prio); } +} + +int +i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, + unsigned int flags, + int prio) +{ + struct dma_fence *excl; - for_each_active(active_mask, idx) { + if (flags & I915_WAIT_ALL) { + struct dma_fence **shared; + unsigned int count, i; int ret; - ret = i915_gem_active_wait_unlocked(&active[idx], - I915_WAIT_INTERRUPTIBLE, - NULL, rps); + ret = reservation_object_get_fences_rcu(obj->resv, + &excl, &count, &shared); if (ret) return ret; + + for (i = 0; i < count; i++) { + fence_set_priority(shared[i], prio); + dma_fence_put(shared[i]); + } + + kfree(shared); + } else { + excl = reservation_object_get_excl_rcu(obj->resv); } + if (excl) { + fence_set_priority(excl, prio); + dma_fence_put(excl); + } return 0; } +/** + * Waits for rendering to the object to be completed + * @obj: i915 gem object + * @flags: how to wait (under a lock, for all rendering or just for writes etc) + * @timeout: how long to wait + * @rps: client (user process) to charge for any waitboosting + */ +int +i915_gem_object_wait(struct drm_i915_gem_object *obj, + unsigned int flags, + long timeout, + struct intel_rps_client *rps) +{ + might_sleep(); +#if IS_ENABLED(CONFIG_LOCKDEP) + GEM_BUG_ON(debug_locks && + !!lockdep_is_held(&obj->base.dev->struct_mutex) != + !!(flags & I915_WAIT_LOCKED)); +#endif + GEM_BUG_ON(timeout < 0); + + timeout = i915_gem_object_wait_reservation(obj->resv, + flags, timeout, + rps); + return timeout < 0 ? timeout : 0; +} + static struct intel_rps_client *to_rps_client(struct drm_file *file) { struct drm_i915_file_private *fpriv = file->driver_priv; @@ -416,7 +548,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, return 0; } - if (obj->madv != I915_MADV_WILLNEED) + if (obj->mm.madv != I915_MADV_WILLNEED) return -EFAULT; if (obj->base.filp == NULL) @@ -426,9 +558,9 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, if (ret) return ret; - ret = i915_gem_object_put_pages(obj); - if (ret) - return ret; + __i915_gem_object_put_pages(obj, I915_MM_NORMAL); + if (obj->mm.pages) + return -EBUSY; /* create a new object */ phys = drm_pci_alloc(obj->base.dev, obj->base.size, align); @@ -438,23 +570,29 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, obj->phys_handle = phys; obj->ops = &i915_gem_phys_ops; - return i915_gem_object_get_pages(obj); + return i915_gem_object_pin_pages(obj); } static int i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_device *dev = obj->base.dev; void *vaddr = obj->phys_handle->vaddr + args->offset; char __user *user_data = u64_to_user_ptr(args->data_ptr); - int ret = 0; + int ret; /* We manually control the domain here and pretend that it * remains coherent i.e. in the GTT domain, like shmem_pwrite. */ - ret = i915_gem_object_wait_rendering(obj, false); + lockdep_assert_held(&obj->base.dev->struct_mutex); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); if (ret) return ret; @@ -516,7 +654,7 @@ i915_gem_create(struct drm_file *file, ret = drm_gem_handle_create(file, &obj->base, &handle); /* drop reference from allocate - handle holds it now */ - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); if (ret) return ret; @@ -548,6 +686,8 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_create *args = data; + i915_gem_flush_free_objects(to_i915(dev)); + return i915_gem_create(file, dev, args->size, &args->handle); } @@ -614,21 +754,24 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, { int ret; - *needs_clflush = 0; + lockdep_assert_held(&obj->base.dev->struct_mutex); + *needs_clflush = 0; if (!i915_gem_object_has_struct_page(obj)) return -ENODEV; - ret = i915_gem_object_wait_rendering(obj, true); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - i915_gem_object_flush_gtt_write_domain(obj); /* If we're not in the cpu read domain, set ourself into the gtt @@ -661,20 +804,25 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, { int ret; + lockdep_assert_held(&obj->base.dev->struct_mutex); + *needs_clflush = 0; if (!i915_gem_object_has_struct_page(obj)) return -ENODEV; - ret = i915_gem_object_wait_rendering(obj, false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - i915_gem_object_flush_gtt_write_domain(obj); /* If we're not in the cpu write domain, set ourself into the @@ -704,7 +852,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, obj->cache_dirty = true; intel_fb_obj_invalidate(obj, ORIGIN_CPU); - obj->dirty = 1; + obj->mm.dirty = true; /* return with the pages pinned */ return 0; @@ -713,32 +861,6 @@ err_unpin: return ret; } -/* Per-page copy function for the shmem pread fastpath. - * Flushes invalid cachelines before reading the target if - * needs_clflush is set. */ -static int -shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, - char __user *user_data, - bool page_do_bit17_swizzling, bool needs_clflush) -{ - char *vaddr; - int ret; - - if (unlikely(page_do_bit17_swizzling)) - return -EINVAL; - - vaddr = kmap_atomic(page); - if (needs_clflush) - drm_clflush_virt_range(vaddr + shmem_page_offset, - page_length); - ret = __copy_to_user_inatomic(user_data, - vaddr + shmem_page_offset, - page_length); - kunmap_atomic(vaddr); - - return ret ? -EFAULT : 0; -} - static void shmem_clflush_swizzled_range(char *addr, unsigned long length, bool swizzled) @@ -764,7 +886,7 @@ shmem_clflush_swizzled_range(char *addr, unsigned long length, /* Only difference to the fast-path function is that this can handle bit17 * and uses non-atomic copy and kmap functions. */ static int -shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, +shmem_pread_slow(struct page *page, int offset, int length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush) { @@ -773,60 +895,130 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, vaddr = kmap(page); if (needs_clflush) - shmem_clflush_swizzled_range(vaddr + shmem_page_offset, - page_length, + shmem_clflush_swizzled_range(vaddr + offset, length, page_do_bit17_swizzling); if (page_do_bit17_swizzling) - ret = __copy_to_user_swizzled(user_data, - vaddr, shmem_page_offset, - page_length); + ret = __copy_to_user_swizzled(user_data, vaddr, offset, length); else - ret = __copy_to_user(user_data, - vaddr + shmem_page_offset, - page_length); + ret = __copy_to_user(user_data, vaddr + offset, length); kunmap(page); return ret ? - EFAULT : 0; } -static inline unsigned long -slow_user_access(struct io_mapping *mapping, - uint64_t page_base, int page_offset, - char __user *user_data, - unsigned long length, bool pwrite) +static int +shmem_pread(struct page *page, int offset, int length, char __user *user_data, + bool page_do_bit17_swizzling, bool needs_clflush) +{ + int ret; + + ret = -ENODEV; + if (!page_do_bit17_swizzling) { + char *vaddr = kmap_atomic(page); + + if (needs_clflush) + drm_clflush_virt_range(vaddr + offset, length); + ret = __copy_to_user_inatomic(user_data, vaddr + offset, length); + kunmap_atomic(vaddr); + } + if (ret == 0) + return 0; + + return shmem_pread_slow(page, offset, length, user_data, + page_do_bit17_swizzling, needs_clflush); +} + +static int +i915_gem_shmem_pread(struct drm_i915_gem_object *obj, + struct drm_i915_gem_pread *args) +{ + char __user *user_data; + u64 remain; + unsigned int obj_do_bit17_swizzling; + unsigned int needs_clflush; + unsigned int idx, offset; + int ret; + + obj_do_bit17_swizzling = 0; + if (i915_gem_object_needs_bit17_swizzle(obj)) + obj_do_bit17_swizzling = BIT(17); + + ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex); + if (ret) + return ret; + + ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); + mutex_unlock(&obj->base.dev->struct_mutex); + if (ret) + return ret; + + remain = args->size; + user_data = u64_to_user_ptr(args->data_ptr); + offset = offset_in_page(args->offset); + for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { + struct page *page = i915_gem_object_get_page(obj, idx); + int length; + + length = remain; + if (offset + length > PAGE_SIZE) + length = PAGE_SIZE - offset; + + ret = shmem_pread(page, offset, length, user_data, + page_to_phys(page) & obj_do_bit17_swizzling, + needs_clflush); + if (ret) + break; + + remain -= length; + user_data += length; + offset = 0; + } + + i915_gem_obj_finish_shmem_access(obj); + return ret; +} + +static inline bool +gtt_user_read(struct io_mapping *mapping, + loff_t base, int offset, + char __user *user_data, int length) { - void __iomem *ioaddr; void *vaddr; - uint64_t unwritten; + unsigned long unwritten; - ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE); /* We can use the cpu mem copy function because this is X86. */ - vaddr = (void __force *)ioaddr + page_offset; - if (pwrite) - unwritten = __copy_from_user(vaddr, user_data, length); - else - unwritten = __copy_to_user(user_data, vaddr, length); - - io_mapping_unmap(ioaddr); + vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base); + unwritten = __copy_to_user_inatomic(user_data, vaddr + offset, length); + io_mapping_unmap_atomic(vaddr); + if (unwritten) { + vaddr = (void __force *) + io_mapping_map_wc(mapping, base, PAGE_SIZE); + unwritten = copy_to_user(user_data, vaddr + offset, length); + io_mapping_unmap(vaddr); + } return unwritten; } static int -i915_gem_gtt_pread(struct drm_device *dev, - struct drm_i915_gem_object *obj, uint64_t size, - uint64_t data_offset, uint64_t data_ptr) +i915_gem_gtt_pread(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pread *args) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; - struct i915_vma *vma; + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct i915_ggtt *ggtt = &i915->ggtt; struct drm_mm_node node; - char __user *user_data; - uint64_t remain; - uint64_t offset; + struct i915_vma *vma; + void __user *user_data; + u64 remain, offset; int ret; - vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); + ret = mutex_lock_interruptible(&i915->drm.struct_mutex); + if (ret) + return ret; + + intel_runtime_pm_get(i915); + vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, + PIN_MAPPABLE | PIN_NONBLOCK); if (!IS_ERR(vma)) { node.start = i915_ggtt_offset(vma); node.allocated = false; @@ -837,35 +1029,21 @@ i915_gem_gtt_pread(struct drm_device *dev, } } if (IS_ERR(vma)) { - ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE); + ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); if (ret) - goto out; - - ret = i915_gem_object_get_pages(obj); - if (ret) { - remove_mappable_node(&node); - goto out; - } - - i915_gem_object_pin_pages(obj); + goto out_unlock; + GEM_BUG_ON(!node.allocated); } ret = i915_gem_object_set_to_gtt_domain(obj, false); if (ret) goto out_unpin; - user_data = u64_to_user_ptr(data_ptr); - remain = size; - offset = data_offset; + mutex_unlock(&i915->drm.struct_mutex); - mutex_unlock(&dev->struct_mutex); - if (likely(!i915.prefault_disable)) { - ret = fault_in_pages_writeable(user_data, remain); - if (ret) { - mutex_lock(&dev->struct_mutex); - goto out_unpin; - } - } + user_data = u64_to_user_ptr(args->data_ptr); + remain = args->size; + offset = args->offset; while (remain > 0) { /* Operation in this page @@ -882,19 +1060,14 @@ i915_gem_gtt_pread(struct drm_device *dev, wmb(); ggtt->base.insert_page(&ggtt->base, i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), - node.start, - I915_CACHE_NONE, 0); + node.start, I915_CACHE_NONE, 0); wmb(); } else { page_base += offset & PAGE_MASK; } - /* This is a slow read/write as it tries to read from - * and write to user memory which may result into page - * faults, and so we cannot perform this under struct_mutex. - */ - if (slow_user_access(&ggtt->mappable, page_base, - page_offset, user_data, - page_length, false)) { + + if (gtt_user_read(&ggtt->mappable, page_base, page_offset, + user_data, page_length)) { ret = -EFAULT; break; } @@ -904,110 +1077,19 @@ i915_gem_gtt_pread(struct drm_device *dev, offset += page_length; } - mutex_lock(&dev->struct_mutex); - if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { - /* The user has modified the object whilst we tried - * reading from it, and we now have no idea what domain - * the pages should be in. As we have just been touching - * them directly, flush everything back to the GTT - * domain. - */ - ret = i915_gem_object_set_to_gtt_domain(obj, false); - } - + mutex_lock(&i915->drm.struct_mutex); out_unpin: if (node.allocated) { wmb(); ggtt->base.clear_range(&ggtt->base, node.start, node.size); - i915_gem_object_unpin_pages(obj); remove_mappable_node(&node); } else { i915_vma_unpin(vma); } -out: - return ret; -} - -static int -i915_gem_shmem_pread(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pread *args, - struct drm_file *file) -{ - char __user *user_data; - ssize_t remain; - loff_t offset; - int shmem_page_offset, page_length, ret = 0; - int obj_do_bit17_swizzling, page_do_bit17_swizzling; - int prefaulted = 0; - int needs_clflush = 0; - struct sg_page_iter sg_iter; - - ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); - if (ret) - return ret; - - obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - user_data = u64_to_user_ptr(args->data_ptr); - offset = args->offset; - remain = args->size; - - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, - offset >> PAGE_SHIFT) { - struct page *page = sg_page_iter_page(&sg_iter); - - if (remain <= 0) - break; - - /* Operation in this page - * - * shmem_page_offset = offset within page in shmem file - * page_length = bytes to copy for this page - */ - shmem_page_offset = offset_in_page(offset); - page_length = remain; - if ((shmem_page_offset + page_length) > PAGE_SIZE) - page_length = PAGE_SIZE - shmem_page_offset; - - page_do_bit17_swizzling = obj_do_bit17_swizzling && - (page_to_phys(page) & (1 << 17)) != 0; - - ret = shmem_pread_fast(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - needs_clflush); - if (ret == 0) - goto next_page; - - mutex_unlock(&dev->struct_mutex); - - if (likely(!i915.prefault_disable) && !prefaulted) { - ret = fault_in_pages_writeable(user_data, remain); - /* Userspace is tricking us, but we've already clobbered - * its pages with the prefault and promised to write the - * data up to the first fault. Hence ignore any errors - * and just continue. */ - (void)ret; - prefaulted = 1; - } - - ret = shmem_pread_slow(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - needs_clflush); - - mutex_lock(&dev->struct_mutex); - - if (ret) - goto out; - -next_page: - remain -= page_length; - user_data += page_length; - offset += page_length; - } - -out: - i915_gem_obj_finish_shmem_access(obj); +out_unlock: + intel_runtime_pm_put(i915); + mutex_unlock(&i915->drm.struct_mutex); return ret; } @@ -1026,7 +1108,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_pread *args = data; struct drm_i915_gem_object *obj; - int ret = 0; + int ret; if (args->size == 0) return 0; @@ -1044,36 +1126,29 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, if (args->offset > obj->base.size || args->size > obj->base.size - args->offset) { ret = -EINVAL; - goto err; + goto out; } trace_i915_gem_object_pread(obj, args->offset, args->size); - ret = __unsafe_wait_rendering(obj, to_rps_client(file), true); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); if (ret) - goto err; + goto out; - ret = i915_mutex_lock_interruptible(dev); + ret = i915_gem_object_pin_pages(obj); if (ret) - goto err; - - ret = i915_gem_shmem_pread(dev, obj, args, file); + goto out; - /* pread for non shmem backed objects */ - if (ret == -EFAULT || ret == -ENODEV) { - intel_runtime_pm_get(to_i915(dev)); - ret = i915_gem_gtt_pread(dev, obj, args->size, - args->offset, args->data_ptr); - intel_runtime_pm_put(to_i915(dev)); - } + ret = i915_gem_shmem_pread(obj, args); + if (ret == -EFAULT || ret == -ENODEV) + ret = i915_gem_gtt_pread(obj, args); + i915_gem_object_unpin_pages(obj); +out: i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); - - return ret; - -err: - i915_gem_object_put_unlocked(obj); return ret; } @@ -1081,51 +1156,52 @@ err: * page faults in the source data */ -static inline int -fast_user_write(struct io_mapping *mapping, - loff_t page_base, int page_offset, - char __user *user_data, - int length) +static inline bool +ggtt_write(struct io_mapping *mapping, + loff_t base, int offset, + char __user *user_data, int length) { - void __iomem *vaddr_atomic; void *vaddr; unsigned long unwritten; - vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); /* We can use the cpu mem copy function because this is X86. */ - vaddr = (void __force*)vaddr_atomic + page_offset; - unwritten = __copy_from_user_inatomic_nocache(vaddr, + vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base); + unwritten = __copy_from_user_inatomic_nocache(vaddr + offset, user_data, length); - io_mapping_unmap_atomic(vaddr_atomic); + io_mapping_unmap_atomic(vaddr); + if (unwritten) { + vaddr = (void __force *) + io_mapping_map_wc(mapping, base, PAGE_SIZE); + unwritten = copy_from_user(vaddr + offset, user_data, length); + io_mapping_unmap(vaddr); + } + return unwritten; } /** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. - * @i915: i915 device private data - * @obj: i915 gem object + * @obj: i915 GEM object * @args: pwrite arguments structure - * @file: drm file pointer */ static int -i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pwrite *args) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_ggtt *ggtt = &i915->ggtt; - struct drm_device *dev = obj->base.dev; - struct i915_vma *vma; struct drm_mm_node node; - uint64_t remain, offset; - char __user *user_data; + struct i915_vma *vma; + u64 remain, offset; + void __user *user_data; int ret; - bool hit_slow_path = false; - if (i915_gem_object_is_tiled(obj)) - return -EFAULT; + ret = mutex_lock_interruptible(&i915->drm.struct_mutex); + if (ret) + return ret; + intel_runtime_pm_get(i915); vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE | PIN_NONBLOCK); if (!IS_ERR(vma)) { @@ -1138,25 +1214,19 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, } } if (IS_ERR(vma)) { - ret = insert_mappable_node(i915, &node, PAGE_SIZE); + ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); if (ret) - goto out; - - ret = i915_gem_object_get_pages(obj); - if (ret) { - remove_mappable_node(&node); - goto out; - } - - i915_gem_object_pin_pages(obj); + goto out_unlock; + GEM_BUG_ON(!node.allocated); } ret = i915_gem_object_set_to_gtt_domain(obj, true); if (ret) goto out_unpin; + mutex_unlock(&i915->drm.struct_mutex); + intel_fb_obj_invalidate(obj, ORIGIN_CPU); - obj->dirty = true; user_data = u64_to_user_ptr(args->data_ptr); offset = args->offset; @@ -1169,8 +1239,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, * page_length = bytes to copy for this page */ u32 page_base = node.start; - unsigned page_offset = offset_in_page(offset); - unsigned page_length = PAGE_SIZE - page_offset; + unsigned int page_offset = offset_in_page(offset); + unsigned int page_length = PAGE_SIZE - page_offset; page_length = remain < page_length ? remain : page_length; if (node.allocated) { wmb(); /* flush the write before we modify the GGTT */ @@ -1187,91 +1257,36 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, * If the object is non-shmem backed, we retry again with the * path that handles page fault. */ - if (fast_user_write(&ggtt->mappable, page_base, - page_offset, user_data, page_length)) { - hit_slow_path = true; - mutex_unlock(&dev->struct_mutex); - if (slow_user_access(&ggtt->mappable, - page_base, - page_offset, user_data, - page_length, true)) { - ret = -EFAULT; - mutex_lock(&dev->struct_mutex); - goto out_flush; - } - - mutex_lock(&dev->struct_mutex); + if (ggtt_write(&ggtt->mappable, page_base, page_offset, + user_data, page_length)) { + ret = -EFAULT; + break; } remain -= page_length; user_data += page_length; offset += page_length; } - -out_flush: - if (hit_slow_path) { - if (ret == 0 && - (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { - /* The user has modified the object whilst we tried - * reading from it, and we now have no idea what domain - * the pages should be in. As we have just been touching - * them directly, flush everything back to the GTT - * domain. - */ - ret = i915_gem_object_set_to_gtt_domain(obj, false); - } - } - intel_fb_obj_flush(obj, false, ORIGIN_CPU); + + mutex_lock(&i915->drm.struct_mutex); out_unpin: if (node.allocated) { wmb(); ggtt->base.clear_range(&ggtt->base, node.start, node.size); - i915_gem_object_unpin_pages(obj); remove_mappable_node(&node); } else { i915_vma_unpin(vma); } -out: +out_unlock: + intel_runtime_pm_put(i915); + mutex_unlock(&i915->drm.struct_mutex); return ret; } -/* Per-page copy function for the shmem pwrite fastpath. - * Flushes invalid cachelines before writing to the target if - * needs_clflush_before is set and flushes out any written cachelines after - * writing if needs_clflush is set. */ -static int -shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, - char __user *user_data, - bool page_do_bit17_swizzling, - bool needs_clflush_before, - bool needs_clflush_after) -{ - char *vaddr; - int ret; - - if (unlikely(page_do_bit17_swizzling)) - return -EINVAL; - - vaddr = kmap_atomic(page); - if (needs_clflush_before) - drm_clflush_virt_range(vaddr + shmem_page_offset, - page_length); - ret = __copy_from_user_inatomic(vaddr + shmem_page_offset, - user_data, page_length); - if (needs_clflush_after) - drm_clflush_virt_range(vaddr + shmem_page_offset, - page_length); - kunmap_atomic(vaddr); - - return ret ? -EFAULT : 0; -} - -/* Only difference to the fast-path function is that this can handle bit17 - * and uses non-atomic copy and kmap functions. */ static int -shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, +shmem_pwrite_slow(struct page *page, int offset, int length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush_before, @@ -1282,124 +1297,114 @@ shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, vaddr = kmap(page); if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) - shmem_clflush_swizzled_range(vaddr + shmem_page_offset, - page_length, + shmem_clflush_swizzled_range(vaddr + offset, length, page_do_bit17_swizzling); if (page_do_bit17_swizzling) - ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, - user_data, - page_length); + ret = __copy_from_user_swizzled(vaddr, offset, user_data, + length); else - ret = __copy_from_user(vaddr + shmem_page_offset, - user_data, - page_length); + ret = __copy_from_user(vaddr + offset, user_data, length); if (needs_clflush_after) - shmem_clflush_swizzled_range(vaddr + shmem_page_offset, - page_length, + shmem_clflush_swizzled_range(vaddr + offset, length, page_do_bit17_swizzling); kunmap(page); return ret ? -EFAULT : 0; } +/* Per-page copy function for the shmem pwrite fastpath. + * Flushes invalid cachelines before writing to the target if + * needs_clflush_before is set and flushes out any written cachelines after + * writing if needs_clflush is set. + */ static int -i915_gem_shmem_pwrite(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, + bool page_do_bit17_swizzling, + bool needs_clflush_before, + bool needs_clflush_after) { - ssize_t remain; - loff_t offset; - char __user *user_data; - int shmem_page_offset, page_length, ret = 0; - int obj_do_bit17_swizzling, page_do_bit17_swizzling; - int hit_slowpath = 0; - unsigned int needs_clflush; - struct sg_page_iter sg_iter; + int ret; - ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); - if (ret) - return ret; + ret = -ENODEV; + if (!page_do_bit17_swizzling) { + char *vaddr = kmap_atomic(page); - obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - user_data = u64_to_user_ptr(args->data_ptr); - offset = args->offset; - remain = args->size; + if (needs_clflush_before) + drm_clflush_virt_range(vaddr + offset, len); + ret = __copy_from_user_inatomic(vaddr + offset, user_data, len); + if (needs_clflush_after) + drm_clflush_virt_range(vaddr + offset, len); - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, - offset >> PAGE_SHIFT) { - struct page *page = sg_page_iter_page(&sg_iter); - int partial_cacheline_write; + kunmap_atomic(vaddr); + } + if (ret == 0) + return ret; - if (remain <= 0) - break; + return shmem_pwrite_slow(page, offset, len, user_data, + page_do_bit17_swizzling, + needs_clflush_before, + needs_clflush_after); +} - /* Operation in this page - * - * shmem_page_offset = offset within page in shmem file - * page_length = bytes to copy for this page - */ - shmem_page_offset = offset_in_page(offset); - - page_length = remain; - if ((shmem_page_offset + page_length) > PAGE_SIZE) - page_length = PAGE_SIZE - shmem_page_offset; - - /* If we don't overwrite a cacheline completely we need to be - * careful to have up-to-date data by first clflushing. Don't - * overcomplicate things and flush the entire patch. */ - partial_cacheline_write = needs_clflush & CLFLUSH_BEFORE && - ((shmem_page_offset | page_length) - & (boot_cpu_data.x86_clflush_size - 1)); - - page_do_bit17_swizzling = obj_do_bit17_swizzling && - (page_to_phys(page) & (1 << 17)) != 0; - - ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - partial_cacheline_write, - needs_clflush & CLFLUSH_AFTER); - if (ret == 0) - goto next_page; - - hit_slowpath = 1; - mutex_unlock(&dev->struct_mutex); - ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - partial_cacheline_write, - needs_clflush & CLFLUSH_AFTER); +static int +i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pwrite *args) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + void __user *user_data; + u64 remain; + unsigned int obj_do_bit17_swizzling; + unsigned int partial_cacheline_write; + unsigned int needs_clflush; + unsigned int offset, idx; + int ret; - mutex_lock(&dev->struct_mutex); + ret = mutex_lock_interruptible(&i915->drm.struct_mutex); + if (ret) + return ret; - if (ret) - goto out; + ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); + mutex_unlock(&i915->drm.struct_mutex); + if (ret) + return ret; -next_page: - remain -= page_length; - user_data += page_length; - offset += page_length; - } + obj_do_bit17_swizzling = 0; + if (i915_gem_object_needs_bit17_swizzle(obj)) + obj_do_bit17_swizzling = BIT(17); -out: - i915_gem_obj_finish_shmem_access(obj); + /* If we don't overwrite a cacheline completely we need to be + * careful to have up-to-date data by first clflushing. Don't + * overcomplicate things and flush the entire patch. + */ + partial_cacheline_write = 0; + if (needs_clflush & CLFLUSH_BEFORE) + partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1; - if (hit_slowpath) { - /* - * Fixup: Flush cpu caches in case we didn't flush the dirty - * cachelines in-line while writing and the object moved - * out of the cpu write domain while we've dropped the lock. - */ - if (!(needs_clflush & CLFLUSH_AFTER) && - obj->base.write_domain != I915_GEM_DOMAIN_CPU) { - if (i915_gem_clflush_object(obj, obj->pin_display)) - needs_clflush |= CLFLUSH_AFTER; - } - } + user_data = u64_to_user_ptr(args->data_ptr); + remain = args->size; + offset = offset_in_page(args->offset); + for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { + struct page *page = i915_gem_object_get_page(obj, idx); + int length; + + length = remain; + if (offset + length > PAGE_SIZE) + length = PAGE_SIZE - offset; + + ret = shmem_pwrite(page, offset, length, user_data, + page_to_phys(page) & obj_do_bit17_swizzling, + (offset | length) & partial_cacheline_write, + needs_clflush & CLFLUSH_AFTER); + if (ret) + break; - if (needs_clflush & CLFLUSH_AFTER) - i915_gem_chipset_flush(to_i915(dev)); + remain -= length; + user_data += length; + offset = 0; + } intel_fb_obj_flush(obj, false, ORIGIN_CPU); + i915_gem_obj_finish_shmem_access(obj); return ret; } @@ -1415,7 +1420,6 @@ int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_pwrite *args = data; struct drm_i915_gem_object *obj; int ret; @@ -1428,13 +1432,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, args->size)) return -EFAULT; - if (likely(!i915.prefault_disable)) { - ret = fault_in_pages_readable(u64_to_user_ptr(args->data_ptr), - args->size); - if (ret) - return -EFAULT; - } - obj = i915_gem_object_lookup(file, args->handle); if (!obj) return -ENOENT; @@ -1448,15 +1445,17 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, trace_i915_gem_object_pwrite(obj, args->offset, args->size); - ret = __unsafe_wait_rendering(obj, to_rps_client(file), false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); if (ret) goto err; - intel_runtime_pm_get(dev_priv); - - ret = i915_mutex_lock_interruptible(dev); + ret = i915_gem_object_pin_pages(obj); if (ret) - goto err_rpm; + goto err; ret = -EFAULT; /* We can only do the GTT pwrite on untiled buffers, as otherwise @@ -1466,30 +1465,23 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, * perspective, requiring manual detiling by the client. */ if (!i915_gem_object_has_struct_page(obj) || - cpu_write_needs_clflush(obj)) { - ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file); + cpu_write_needs_clflush(obj)) /* Note that the gtt paths might fail with non-page-backed user * pointers (e.g. gtt mappings when moving data between - * textures). Fallback to the shmem path in that case. */ - } + * textures). Fallback to the shmem path in that case. + */ + ret = i915_gem_gtt_pwrite_fast(obj, args); if (ret == -EFAULT || ret == -ENOSPC) { if (obj->phys_handle) ret = i915_gem_phys_pwrite(obj, args, file); else - ret = i915_gem_shmem_pwrite(dev, obj, args, file); + ret = i915_gem_shmem_pwrite(obj, args); } - i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); - intel_runtime_pm_put(dev_priv); - - return ret; - -err_rpm: - intel_runtime_pm_put(dev_priv); + i915_gem_object_unpin_pages(obj); err: - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return ret; } @@ -1500,6 +1492,30 @@ write_origin(struct drm_i915_gem_object *obj, unsigned domain) obj->frontbuffer_ggtt_origin : ORIGIN_CPU); } +static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *i915; + struct list_head *list; + struct i915_vma *vma; + + list_for_each_entry(vma, &obj->vma_list, obj_link) { + if (!i915_vma_is_ggtt(vma)) + continue; + + if (i915_vma_is_active(vma)) + continue; + + if (!drm_mm_node_allocated(&vma->node)) + continue; + + list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + } + + i915 = to_i915(obj->base.dev); + list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; + list_move_tail(&obj->global_link, list); +} + /** * Called when user space prepares to use an object with the CPU, either * through the mmap ioctl's mapping or a GTT mapping. @@ -1515,7 +1531,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; uint32_t read_domains = args->read_domains; uint32_t write_domain = args->write_domain; - int ret; + int err; /* Only handle setting domains to types used by the CPU. */ if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) @@ -1535,29 +1551,48 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, * We will repeat the flush holding the lock in the normal manner * to catch cases where we are gazumped. */ - ret = __unsafe_wait_rendering(obj, to_rps_client(file), !write_domain); - if (ret) - goto err; + err = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + (write_domain ? I915_WAIT_ALL : 0), + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); + if (err) + goto out; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - goto err; + /* Flush and acquire obj->pages so that we are coherent through + * direct access in memory with previous cached writes through + * shmemfs and that our cache domain tracking remains valid. + * For example, if the obj->filp was moved to swap without us + * being notified and releasing the pages, we would mistakenly + * continue to assume that the obj remained out of the CPU cached + * domain. + */ + err = i915_gem_object_pin_pages(obj); + if (err) + goto out; + + err = i915_mutex_lock_interruptible(dev); + if (err) + goto out_unpin; if (read_domains & I915_GEM_DOMAIN_GTT) - ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); + err = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); else - ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); + err = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); + + /* And bump the LRU for this access */ + i915_gem_object_bump_inactive_ggtt(obj); + + mutex_unlock(&dev->struct_mutex); if (write_domain != 0) intel_fb_obj_invalidate(obj, write_origin(obj, write_domain)); +out_unpin: + i915_gem_object_unpin_pages(obj); +out: i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); - return ret; - -err: - i915_gem_object_put_unlocked(obj); - return ret; + return err; } /** @@ -1587,7 +1622,7 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, } } - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return err; } @@ -1633,7 +1668,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, * pages from. */ if (!obj->base.filp) { - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return -EINVAL; } @@ -1645,7 +1680,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct vm_area_struct *vma; if (down_write_killable(&mm->mmap_sem)) { - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return -EINTR; } vma = find_vma(mm, addr); @@ -1659,7 +1694,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, /* This may race, but that's ok, it only gets set */ WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU); } - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); if (IS_ERR((void *)addr)) return addr; @@ -1771,7 +1806,14 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) * repeat the flush holding the lock in the normal manner to catch cases * where we are gazumped. */ - ret = __unsafe_wait_rendering(obj, NULL, !write); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT, + NULL); + if (ret) + goto err; + + ret = i915_gem_object_pin_pages(obj); if (ret) goto err; @@ -1782,7 +1824,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) goto err_rpm; /* Access to snoopable pages through the GTT is incoherent. */ - if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { + if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) { ret = -EFAULT; goto err_unlock; } @@ -1804,7 +1846,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) /* Use a partial view if it is bigger than available space */ chunk_size = MIN_CHUNK_PAGES; if (i915_gem_object_is_tiled(obj)) - chunk_size = max(chunk_size, tile_row_pages(obj)); + chunk_size = roundup(chunk_size, tile_row_pages(obj)); memset(&view, 0, sizeof(view)); view.type = I915_GGTT_VIEW_PARTIAL; @@ -1839,22 +1881,25 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) if (ret) goto err_unpin; + /* Mark as being mmapped into userspace for later revocation */ + assert_rpm_wakelock_held(dev_priv); + if (list_empty(&obj->userfault_link)) + list_add(&obj->userfault_link, &dev_priv->mm.userfault_list); + /* Finally, remap it using the new GTT offset */ ret = remap_io_mapping(area, area->vm_start + (vma->ggtt_view.params.partial.offset << PAGE_SHIFT), (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT, min_t(u64, vma->size, area->vm_end - area->vm_start), &ggtt->mappable); - if (ret) - goto err_unpin; - obj->fault_mappable = true; err_unpin: __i915_vma_unpin(vma); err_unlock: mutex_unlock(&dev->struct_mutex); err_rpm: intel_runtime_pm_put(dev_priv); + i915_gem_object_unpin_pages(obj); err: switch (ret) { case -EIO: @@ -1916,15 +1961,23 @@ err: void i915_gem_release_mmap(struct drm_i915_gem_object *obj) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); + /* Serialisation between user GTT access and our code depends upon * revoking the CPU's PTE whilst the mutex is held. The next user * pagefault then has to wait until we release the mutex. + * + * Note that RPM complicates somewhat by adding an additional + * requirement that operations to the GGTT be made holding the RPM + * wakeref. */ - lockdep_assert_held(&obj->base.dev->struct_mutex); + lockdep_assert_held(&i915->drm.struct_mutex); + intel_runtime_pm_get(i915); - if (!obj->fault_mappable) - return; + if (list_empty(&obj->userfault_link)) + goto out; + list_del_init(&obj->userfault_link); drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->anon_inode->i_mapping); @@ -1937,16 +1990,45 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) */ wmb(); - obj->fault_mappable = false; +out: + intel_runtime_pm_put(i915); } -void -i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) +void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv) { - struct drm_i915_gem_object *obj; + struct drm_i915_gem_object *obj, *on; + int i; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) - i915_gem_release_mmap(obj); + /* + * Only called during RPM suspend. All users of the userfault_list + * must be holding an RPM wakeref to ensure that this can not + * run concurrently with themselves (and use the struct_mutex for + * protection between themselves). + */ + + list_for_each_entry_safe(obj, on, + &dev_priv->mm.userfault_list, userfault_link) { + list_del_init(&obj->userfault_link); + drm_vma_node_unmap(&obj->base.vma_node, + obj->base.dev->anon_inode->i_mapping); + } + + /* The fence will be lost when the device powers down. If any were + * in use by hardware (i.e. they are pinned), we should not be powering + * down! All other fences will be reacquired by the user upon waking. + */ + for (i = 0; i < dev_priv->num_fence_regs; i++) { + struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; + + if (WARN_ON(reg->pin_count)) + continue; + + if (!reg->vma) + continue; + + GEM_BUG_ON(!list_empty(®->vma->obj->userfault_link)); + reg->dirty = true; + } } /** @@ -2060,7 +2142,7 @@ i915_gem_mmap_gtt(struct drm_file *file, if (ret == 0) *offset = drm_vma_node_offset_addr(&obj->base.vma_node); - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return ret; } @@ -2103,16 +2185,18 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj) * backing pages, *now*. */ shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); - obj->madv = __I915_MADV_PURGED; + obj->mm.madv = __I915_MADV_PURGED; } /* Try to discard unwanted pages */ -static void -i915_gem_object_invalidate(struct drm_i915_gem_object *obj) +void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj) { struct address_space *mapping; - switch (obj->madv) { + lockdep_assert_held(&obj->mm.lock); + GEM_BUG_ON(obj->mm.pages); + + switch (obj->mm.madv) { case I915_MADV_DONTNEED: i915_gem_object_truncate(obj); case __I915_MADV_PURGED: @@ -2127,82 +2211,83 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj) } static void -i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) +i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct sgt_iter sgt_iter; struct page *page; - int ret; - BUG_ON(obj->madv == __I915_MADV_PURGED); + __i915_gem_object_release_shmem(obj, pages); - ret = i915_gem_object_set_to_cpu_domain(obj, true); - if (WARN_ON(ret)) { - /* In the event of a disaster, abandon all caches and - * hope for the best. - */ - i915_gem_clflush_object(obj, true); - obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; - } - - i915_gem_gtt_finish_object(obj); + i915_gem_gtt_finish_pages(obj, pages); if (i915_gem_object_needs_bit17_swizzle(obj)) - i915_gem_object_save_bit_17_swizzle(obj); - - if (obj->madv == I915_MADV_DONTNEED) - obj->dirty = 0; + i915_gem_object_save_bit_17_swizzle(obj, pages); - for_each_sgt_page(page, sgt_iter, obj->pages) { - if (obj->dirty) + for_each_sgt_page(page, sgt_iter, pages) { + if (obj->mm.dirty) set_page_dirty(page); - if (obj->madv == I915_MADV_WILLNEED) + if (obj->mm.madv == I915_MADV_WILLNEED) mark_page_accessed(page); put_page(page); } - obj->dirty = 0; + obj->mm.dirty = false; - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(pages); + kfree(pages); } -int -i915_gem_object_put_pages(struct drm_i915_gem_object *obj) +static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) { - const struct drm_i915_gem_object_ops *ops = obj->ops; + struct radix_tree_iter iter; + void **slot; - if (obj->pages == NULL) - return 0; + radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) + radix_tree_delete(&obj->mm.get_page.radix, iter.index); +} - if (obj->pages_pin_count) - return -EBUSY; +void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, + enum i915_mm_subclass subclass) +{ + struct sg_table *pages; + + if (i915_gem_object_has_pinned_pages(obj)) + return; GEM_BUG_ON(obj->bind_count); + if (!READ_ONCE(obj->mm.pages)) + return; + + /* May be called by shrinker from within get_pages() (on another bo) */ + mutex_lock_nested(&obj->mm.lock, subclass); + if (unlikely(atomic_read(&obj->mm.pages_pin_count))) + goto unlock; /* ->put_pages might need to allocate memory for the bit17 swizzle * array, hence protect them from being reaped by removing them from gtt * lists early. */ - list_del(&obj->global_list); + pages = fetch_and_zero(&obj->mm.pages); + GEM_BUG_ON(!pages); - if (obj->mapping) { + if (obj->mm.mapping) { void *ptr; - ptr = ptr_mask_bits(obj->mapping); + ptr = ptr_mask_bits(obj->mm.mapping); if (is_vmalloc_addr(ptr)) vunmap(ptr); else kunmap(kmap_to_page(ptr)); - obj->mapping = NULL; + obj->mm.mapping = NULL; } - ops->put_pages(obj); - obj->pages = NULL; + __i915_gem_object_reset_page_iter(obj); - i915_gem_object_invalidate(obj); - - return 0; + obj->ops->put_pages(obj, pages); +unlock: + mutex_unlock(&obj->mm.lock); } static unsigned int swiotlb_max_size(void) @@ -2214,7 +2299,31 @@ static unsigned int swiotlb_max_size(void) #endif } -static int +static void i915_sg_trim(struct sg_table *orig_st) +{ + struct sg_table new_st; + struct scatterlist *sg, *new_sg; + unsigned int i; + + if (orig_st->nents == orig_st->orig_nents) + return; + + if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL)) + return; + + new_sg = new_st.sgl; + for_each_sg(orig_st->sgl, sg, orig_st->nents, i) { + sg_set_page(new_sg, sg_page(sg), sg->length, 0); + /* called before being DMA mapped, no need to copy sg->dma_* */ + new_sg = sg_next(new_sg); + } + + sg_free_table(orig_st); + + *orig_st = new_st; +} + +static struct sg_table * i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); @@ -2233,8 +2342,8 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) * wasn't in the GTT, there shouldn't be any way it could have been in * a GPU cache */ - BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); - BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); + GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); + GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); max_segment = swiotlb_max_size(); if (!max_segment) @@ -2242,12 +2351,12 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) st = kmalloc(sizeof(*st), GFP_KERNEL); if (st == NULL) - return -ENOMEM; + return ERR_PTR(-ENOMEM); page_count = obj->base.size / PAGE_SIZE; if (sg_alloc_table(st, page_count, GFP_KERNEL)) { kfree(st); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } /* Get the list of pages out of our struct file. They'll be pinned @@ -2278,7 +2387,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) page = shmem_read_mapping_page(mapping, i); if (IS_ERR(page)) { ret = PTR_ERR(page); - goto err_pages; + goto err_sg; } } if (!i || @@ -2298,23 +2407,22 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) } if (sg) /* loop terminated early; short sg table */ sg_mark_end(sg); - obj->pages = st; - ret = i915_gem_gtt_prepare_object(obj); + /* Trim unused sg entries to avoid wasting memory. */ + i915_sg_trim(st); + + ret = i915_gem_gtt_prepare_pages(obj, st); if (ret) goto err_pages; if (i915_gem_object_needs_bit17_swizzle(obj)) - i915_gem_object_do_bit_17_swizzle(obj); + i915_gem_object_do_bit_17_swizzle(obj, st); - if (i915_gem_object_is_tiled(obj) && - dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) - i915_gem_object_pin_pages(obj); - - return 0; + return st; -err_pages: +err_sg: sg_mark_end(sg); +err_pages: for_each_sgt_page(page, sgt_iter, st) put_page(page); sg_free_table(st); @@ -2331,43 +2439,73 @@ err_pages: if (ret == -ENOSPC) ret = -ENOMEM; - return ret; + return ERR_PTR(ret); } -/* Ensure that the associated pages are gathered from the backing storage - * and pinned into our object. i915_gem_object_get_pages() may be called - * multiple times before they are released by a single call to - * i915_gem_object_put_pages() - once the pages are no longer referenced - * either as a result of memory pressure (reaping pages under the shrinker) - * or as the object is itself released. - */ -int -i915_gem_object_get_pages(struct drm_i915_gem_object *obj) +void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - const struct drm_i915_gem_object_ops *ops = obj->ops; - int ret; + lockdep_assert_held(&obj->mm.lock); - if (obj->pages) - return 0; + obj->mm.get_page.sg_pos = pages->sgl; + obj->mm.get_page.sg_idx = 0; + + obj->mm.pages = pages; + + if (i915_gem_object_is_tiled(obj) && + to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES) { + GEM_BUG_ON(obj->mm.quirked); + __i915_gem_object_pin_pages(obj); + obj->mm.quirked = true; + } +} + +static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) +{ + struct sg_table *pages; + + GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); - if (obj->madv != I915_MADV_WILLNEED) { + if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { DRM_DEBUG("Attempting to obtain a purgeable object\n"); return -EFAULT; } - BUG_ON(obj->pages_pin_count); + pages = obj->ops->get_pages(obj); + if (unlikely(IS_ERR(pages))) + return PTR_ERR(pages); - ret = ops->get_pages(obj); - if (ret) - return ret; + __i915_gem_object_set_pages(obj, pages); + return 0; +} - list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); +/* Ensure that the associated pages are gathered from the backing storage + * and pinned into our object. i915_gem_object_pin_pages() may be called + * multiple times before they are released by a single call to + * i915_gem_object_unpin_pages() - once the pages are no longer referenced + * either as a result of memory pressure (reaping pages under the shrinker) + * or as the object is itself released. + */ +int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) +{ + int err; - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; + err = mutex_lock_interruptible(&obj->mm.lock); + if (err) + return err; - return 0; + if (unlikely(!obj->mm.pages)) { + err = ____i915_gem_object_get_pages(obj); + if (err) + goto unlock; + + smp_mb__before_atomic(); + } + atomic_inc(&obj->mm.pages_pin_count); + +unlock: + mutex_unlock(&obj->mm.lock); + return err; } /* The 'mapping' part of i915_gem_object_pin_map() below */ @@ -2375,7 +2513,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, enum i915_map_type type) { unsigned long n_pages = obj->base.size >> PAGE_SHIFT; - struct sg_table *sgt = obj->pages; + struct sg_table *sgt = obj->mm.pages; struct sgt_iter sgt_iter; struct page *page; struct page *stack_pages[32]; @@ -2426,21 +2564,31 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, void *ptr; int ret; - lockdep_assert_held(&obj->base.dev->struct_mutex); GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); - ret = i915_gem_object_get_pages(obj); + ret = mutex_lock_interruptible(&obj->mm.lock); if (ret) return ERR_PTR(ret); - i915_gem_object_pin_pages(obj); - pinned = obj->pages_pin_count > 1; + pinned = true; + if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { + if (unlikely(!obj->mm.pages)) { + ret = ____i915_gem_object_get_pages(obj); + if (ret) + goto err_unlock; - ptr = ptr_unpack_bits(obj->mapping, has_type); + smp_mb__before_atomic(); + } + atomic_inc(&obj->mm.pages_pin_count); + pinned = false; + } + GEM_BUG_ON(!obj->mm.pages); + + ptr = ptr_unpack_bits(obj->mm.mapping, has_type); if (ptr && has_type != type) { if (pinned) { ret = -EBUSY; - goto err; + goto err_unpin; } if (is_vmalloc_addr(ptr)) @@ -2448,59 +2596,28 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, else kunmap(kmap_to_page(ptr)); - ptr = obj->mapping = NULL; + ptr = obj->mm.mapping = NULL; } if (!ptr) { ptr = i915_gem_object_map(obj, type); if (!ptr) { ret = -ENOMEM; - goto err; + goto err_unpin; } - obj->mapping = ptr_pack_bits(ptr, type); + obj->mm.mapping = ptr_pack_bits(ptr, type); } +out_unlock: + mutex_unlock(&obj->mm.lock); return ptr; -err: - i915_gem_object_unpin_pages(obj); - return ERR_PTR(ret); -} - -static void -i915_gem_object_retire__write(struct i915_gem_active *active, - struct drm_i915_gem_request *request) -{ - struct drm_i915_gem_object *obj = - container_of(active, struct drm_i915_gem_object, last_write); - - intel_fb_obj_flush(obj, true, ORIGIN_CS); -} - -static void -i915_gem_object_retire__read(struct i915_gem_active *active, - struct drm_i915_gem_request *request) -{ - int idx = request->engine->id; - struct drm_i915_gem_object *obj = - container_of(active, struct drm_i915_gem_object, last_read[idx]); - - GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx)); - - i915_gem_object_clear_active(obj, idx); - if (i915_gem_object_is_active(obj)) - return; - - /* Bump our place on the bound list to keep it roughly in LRU order - * so that we don't steal from recently used but inactive objects - * (unless we are forced to ofc!) - */ - if (obj->bind_count) - list_move_tail(&obj->global_list, - &request->i915->mm.bound_list); - - i915_gem_object_put(obj); +err_unpin: + atomic_dec(&obj->mm.pages_pin_count); +err_unlock: + ptr = ERR_PTR(ret); + goto out_unlock; } static bool i915_context_is_banned(const struct i915_gem_context *ctx) @@ -2547,13 +2664,10 @@ i915_gem_find_active_request(struct intel_engine_cs *engine) * extra delay for a recent interrupt is pointless. Hence, we do * not need an engine->irq_seqno_barrier() before the seqno reads. */ - list_for_each_entry(request, &engine->request_list, link) { - if (i915_gem_request_completed(request)) + list_for_each_entry(request, &engine->timeline->requests, link) { + if (__i915_gem_request_completed(request)) continue; - if (!i915_sw_fence_done(&request->submit)) - break; - return request; } @@ -2581,6 +2695,7 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request; struct i915_gem_context *incomplete_ctx; + struct intel_timeline *timeline; bool ring_hung; if (engine->irq_seqno_barrier) @@ -2599,7 +2714,7 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) return; DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n", - engine->name, request->fence.seqno); + engine->name, request->global_seqno); /* Setup the CS to resume from the breadcrumb of the hung request */ engine->reset_hw(engine, request); @@ -2616,9 +2731,13 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) if (i915_gem_context_is_default(incomplete_ctx)) return; - list_for_each_entry_continue(request, &engine->request_list, link) + list_for_each_entry_continue(request, &engine->timeline->requests, link) if (request->ctx == incomplete_ctx) reset_request(request); + + timeline = i915_gem_context_lookup_timeline(incomplete_ctx, engine); + list_for_each_entry(request, &timeline->requests, link) + reset_request(request); } void i915_gem_reset(struct drm_i915_private *dev_priv) @@ -2626,12 +2745,14 @@ void i915_gem_reset(struct drm_i915_private *dev_priv) struct intel_engine_cs *engine; enum intel_engine_id id; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + i915_gem_retire_requests(dev_priv); for_each_engine(engine, dev_priv, id) i915_gem_reset_engine(engine); - i915_gem_restore_fences(&dev_priv->drm); + i915_gem_restore_fences(dev_priv); if (dev_priv->gt.awake) { intel_sanitize_gt_powersave(dev_priv); @@ -2643,6 +2764,8 @@ void i915_gem_reset(struct drm_i915_private *dev_priv) static void nop_submit_request(struct drm_i915_gem_request *request) { + i915_gem_request_submit(request); + intel_engine_init_global_seqno(request->engine, request->global_seqno); } static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) @@ -2653,7 +2776,8 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) * (lockless) lookup doesn't try and wait upon the request as we * reset it. */ - intel_engine_init_seqno(engine, engine->last_submitted_seqno); + intel_engine_init_global_seqno(engine, + intel_engine_last_submit(engine)); /* * Clear the execlists queue up before freeing the requests, as those @@ -2662,15 +2786,18 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) */ if (i915.enable_execlists) { - spin_lock(&engine->execlist_lock); - INIT_LIST_HEAD(&engine->execlist_queue); + unsigned long flags; + + spin_lock_irqsave(&engine->timeline->lock, flags); + i915_gem_request_put(engine->execlist_port[0].request); i915_gem_request_put(engine->execlist_port[1].request); memset(engine->execlist_port, 0, sizeof(engine->execlist_port)); - spin_unlock(&engine->execlist_lock); - } + engine->execlist_queue = RB_ROOT; + engine->execlist_first = NULL; - engine->i915->gt.active_engines &= ~intel_engine_flag(engine); + spin_unlock_irqrestore(&engine->timeline->lock, flags); + } } void i915_gem_set_wedged(struct drm_i915_private *dev_priv) @@ -2727,7 +2854,14 @@ i915_gem_idle_work_handler(struct work_struct *work) if (!READ_ONCE(dev_priv->gt.awake)) return; - if (READ_ONCE(dev_priv->gt.active_engines)) + /* + * Wait for last execlists context complete, but bail out in case a + * new request is submitted. + */ + wait_for(READ_ONCE(dev_priv->gt.active_requests) || + intel_execlists_idle(dev_priv), 10); + + if (READ_ONCE(dev_priv->gt.active_requests)) return; rearm_hangcheck = @@ -2741,9 +2875,19 @@ i915_gem_idle_work_handler(struct work_struct *work) goto out_rearm; } - if (dev_priv->gt.active_engines) + /* + * New request retired after this work handler started, extend active + * period until next instance of the work. + */ + if (work_pending(work)) + goto out_unlock; + + if (dev_priv->gt.active_requests) goto out_unlock; + if (wait_for(intel_execlists_idle(dev_priv), 10)) + DRM_ERROR("Timeout waiting for engines to idle\n"); + for_each_engine(engine, dev_priv, id) i915_gem_batch_pool_fini(&engine->batch_pool); @@ -2774,9 +2918,26 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link) if (vma->vm->file == fpriv) i915_vma_close(vma); + + if (i915_gem_object_is_active(obj) && + !i915_gem_object_has_active_reference(obj)) { + i915_gem_object_set_active_reference(obj); + i915_gem_object_get(obj); + } mutex_unlock(&obj->base.dev->struct_mutex); } +static unsigned long to_wait_timeout(s64 timeout_ns) +{ + if (timeout_ns < 0) + return MAX_SCHEDULE_TIMEOUT; + + if (timeout_ns == 0) + return 0; + + return nsecs_to_jiffies_timeout(timeout_ns); +} + /** * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT * @dev: drm device pointer @@ -2805,10 +2966,9 @@ int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_wait *args = data; - struct intel_rps_client *rps = to_rps_client(file); struct drm_i915_gem_object *obj; - unsigned long active; - int idx, ret = 0; + ktime_t start; + long ret; if (args->flags != 0) return -EINVAL; @@ -2817,134 +2977,29 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (!obj) return -ENOENT; - active = __I915_BO_ACTIVE(obj); - for_each_active(active, idx) { - s64 *timeout = args->timeout_ns >= 0 ? &args->timeout_ns : NULL; - ret = i915_gem_active_wait_unlocked(&obj->last_read[idx], - I915_WAIT_INTERRUPTIBLE, - timeout, rps); - if (ret) - break; - } - - i915_gem_object_put_unlocked(obj); - return ret; -} - -static void __i915_vma_iounmap(struct i915_vma *vma) -{ - GEM_BUG_ON(i915_vma_is_pinned(vma)); - - if (vma->iomap == NULL) - return; - - io_mapping_unmap(vma->iomap); - vma->iomap = NULL; -} - -int i915_vma_unbind(struct i915_vma *vma) -{ - struct drm_i915_gem_object *obj = vma->obj; - unsigned long active; - int ret; - - /* First wait upon any activity as retiring the request may - * have side-effects such as unpinning or even unbinding this vma. - */ - active = i915_vma_get_active(vma); - if (active) { - int idx; - - /* When a closed VMA is retired, it is unbound - eek. - * In order to prevent it from being recursively closed, - * take a pin on the vma so that the second unbind is - * aborted. - */ - __i915_vma_pin(vma); - - for_each_active(active, idx) { - ret = i915_gem_active_retire(&vma->last_read[idx], - &vma->vm->dev->struct_mutex); - if (ret) - break; - } - - __i915_vma_unpin(vma); - if (ret) - return ret; - - GEM_BUG_ON(i915_vma_is_active(vma)); - } - - if (i915_vma_is_pinned(vma)) - return -EBUSY; - - if (!drm_mm_node_allocated(&vma->node)) - goto destroy; - - GEM_BUG_ON(obj->bind_count == 0); - GEM_BUG_ON(!obj->pages); - - if (i915_vma_is_map_and_fenceable(vma)) { - /* release the fence reg _after_ flushing */ - ret = i915_vma_put_fence(vma); - if (ret) - return ret; + start = ktime_get(); - /* Force a pagefault for domain tracking on next user access */ - i915_gem_release_mmap(obj); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL, + to_wait_timeout(args->timeout_ns), + to_rps_client(file)); - __i915_vma_iounmap(vma); - vma->flags &= ~I915_VMA_CAN_FENCE; + if (args->timeout_ns > 0) { + args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); + if (args->timeout_ns < 0) + args->timeout_ns = 0; } - if (likely(!vma->vm->closed)) { - trace_i915_vma_unbind(vma); - vma->vm->unbind_vma(vma); - } - vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); - - drm_mm_remove_node(&vma->node); - list_move_tail(&vma->vm_link, &vma->vm->unbound_list); - - if (vma->pages != obj->pages) { - GEM_BUG_ON(!vma->pages); - sg_free_table(vma->pages); - kfree(vma->pages); - } - vma->pages = NULL; - - /* Since the unbound list is global, only move to that list if - * no more VMAs exist. */ - if (--obj->bind_count == 0) - list_move_tail(&obj->global_list, - &to_i915(obj->base.dev)->mm.unbound_list); - - /* And finally now the object is completely decoupled from this vma, - * we can drop its hold on the backing storage and allow it to be - * reaped by the shrinker. - */ - i915_gem_object_unpin_pages(obj); - -destroy: - if (unlikely(i915_vma_is_closed(vma))) - i915_vma_destroy(vma); - - return 0; + i915_gem_object_put(obj); + return ret; } -int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, - unsigned int flags) +static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags) { - struct intel_engine_cs *engine; - enum intel_engine_id id; - int ret; + int ret, i; - for_each_engine(engine, dev_priv, id) { - if (engine->last_context == NULL) - continue; - - ret = intel_engine_idle(engine, flags); + for (i = 0; i < ARRAY_SIZE(tl->engine); i++) { + ret = i915_gem_active_wait(&tl->engine[i].last_request, flags); if (ret) return ret; } @@ -2952,190 +3007,45 @@ int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, return 0; } -static bool i915_gem_valid_gtt_space(struct i915_vma *vma, - unsigned long cache_level) +int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags) { - struct drm_mm_node *gtt_space = &vma->node; - struct drm_mm_node *other; - - /* - * On some machines we have to be careful when putting differing types - * of snoopable memory together to avoid the prefetcher crossing memory - * domains and dying. During vm initialisation, we decide whether or not - * these constraints apply and set the drm_mm.color_adjust - * appropriately. - */ - if (vma->vm->mm.color_adjust == NULL) - return true; - - if (!drm_mm_node_allocated(gtt_space)) - return true; - - if (list_empty(>t_space->node_list)) - return true; - - other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); - if (other->allocated && !other->hole_follows && other->color != cache_level) - return false; - - other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); - if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) - return false; - - return true; -} - -/** - * i915_vma_insert - finds a slot for the vma in its address space - * @vma: the vma - * @size: requested size in bytes (can be larger than the VMA) - * @alignment: required alignment - * @flags: mask of PIN_* flags to use - * - * First we try to allocate some free space that meets the requirements for - * the VMA. Failiing that, if the flags permit, it will evict an old VMA, - * preferrably the oldest idle entry to make room for the new VMA. - * - * Returns: - * 0 on success, negative error code otherwise. - */ -static int -i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) -{ - struct drm_i915_private *dev_priv = to_i915(vma->vm->dev); - struct drm_i915_gem_object *obj = vma->obj; - u64 start, end; int ret; - GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); - GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); + if (flags & I915_WAIT_LOCKED) { + struct i915_gem_timeline *tl; - size = max(size, vma->size); - if (flags & PIN_MAPPABLE) - size = i915_gem_get_ggtt_size(dev_priv, size, - i915_gem_object_get_tiling(obj)); + lockdep_assert_held(&i915->drm.struct_mutex); - alignment = max(max(alignment, vma->display_alignment), - i915_gem_get_ggtt_alignment(dev_priv, size, - i915_gem_object_get_tiling(obj), - flags & PIN_MAPPABLE)); - - start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; - - end = vma->vm->total; - if (flags & PIN_MAPPABLE) - end = min_t(u64, end, dev_priv->ggtt.mappable_end); - if (flags & PIN_ZONE_4G) - end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE); - - /* If binding the object/GGTT view requires more space than the entire - * aperture has, reject it early before evicting everything in a vain - * attempt to find space. - */ - if (size > end) { - DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n", - size, obj->base.size, - flags & PIN_MAPPABLE ? "mappable" : "total", - end); - return -E2BIG; - } - - ret = i915_gem_object_get_pages(obj); - if (ret) - return ret; - - i915_gem_object_pin_pages(obj); - - if (flags & PIN_OFFSET_FIXED) { - u64 offset = flags & PIN_OFFSET_MASK; - if (offset & (alignment - 1) || offset > end - size) { - ret = -EINVAL; - goto err_unpin; - } - - vma->node.start = offset; - vma->node.size = size; - vma->node.color = obj->cache_level; - ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); - if (ret) { - ret = i915_gem_evict_for_vma(vma); - if (ret == 0) - ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); + list_for_each_entry(tl, &i915->gt.timelines, link) { + ret = wait_for_timeline(tl, flags); if (ret) - goto err_unpin; + return ret; } } else { - u32 search_flag, alloc_flag; - - if (flags & PIN_HIGH) { - search_flag = DRM_MM_SEARCH_BELOW; - alloc_flag = DRM_MM_CREATE_TOP; - } else { - search_flag = DRM_MM_SEARCH_DEFAULT; - alloc_flag = DRM_MM_CREATE_DEFAULT; - } - - /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks, - * so we know that we always have a minimum alignment of 4096. - * The drm_mm range manager is optimised to return results - * with zero alignment, so where possible use the optimal - * path. - */ - if (alignment <= 4096) - alignment = 0; - -search_free: - ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm, - &vma->node, - size, alignment, - obj->cache_level, - start, end, - search_flag, - alloc_flag); - if (ret) { - ret = i915_gem_evict_something(vma->vm, size, alignment, - obj->cache_level, - start, end, - flags); - if (ret == 0) - goto search_free; - - goto err_unpin; - } - - GEM_BUG_ON(vma->node.start < start); - GEM_BUG_ON(vma->node.start + vma->node.size > end); + ret = wait_for_timeline(&i915->gt.global_timeline, flags); + if (ret) + return ret; } - GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); - - list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); - obj->bind_count++; return 0; - -err_unpin: - i915_gem_object_unpin_pages(obj); - return ret; } -bool -i915_gem_clflush_object(struct drm_i915_gem_object *obj, - bool force) +void i915_gem_clflush_object(struct drm_i915_gem_object *obj, + bool force) { /* If we don't have a page list set up, then we're not pinned * to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ - if (obj->pages == NULL) - return false; + if (!obj->mm.pages) + return; /* * Stolen memory is always coherent with the GPU as it is explicitly * marked as wc by the system, or the system is cache-coherent. */ if (obj->stolen || obj->phys_handle) - return false; + return; /* If the GPU is snooping the contents of the CPU cache, * we do not need to manually clear the CPU cache lines. However, @@ -3147,14 +3057,12 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj, */ if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) { obj->cache_dirty = true; - return false; + return; } trace_i915_gem_object_clflush(obj); - drm_clflush_sg(obj->pages); + drm_clflush_sg(obj->mm.pages); obj->cache_dirty = false; - - return true; } /** Flushes the GTT write domain for the object if it's dirty. */ @@ -3200,9 +3108,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) return; - if (i915_gem_clflush_object(obj, obj->pin_display)) - i915_gem_chipset_flush(to_i915(obj->base.dev)); - + i915_gem_clflush_object(obj, obj->pin_display); intel_fb_obj_flush(obj, false, ORIGIN_CPU); obj->base.write_domain = 0; @@ -3211,24 +3117,6 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) I915_GEM_DOMAIN_CPU); } -static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) -{ - struct i915_vma *vma; - - list_for_each_entry(vma, &obj->vma_list, obj_link) { - if (!i915_vma_is_ggtt(vma)) - continue; - - if (i915_vma_is_active(vma)) - continue; - - if (!drm_mm_node_allocated(&vma->node)) - continue; - - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); - } -} - /** * Moves a single object to the GTT read, and possibly write domain. * @obj: object to act on @@ -3243,7 +3131,14 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) uint32_t old_write_domain, old_read_domains; int ret; - ret = i915_gem_object_wait_rendering(obj, !write); + lockdep_assert_held(&obj->base.dev->struct_mutex); + + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + (write ? I915_WAIT_ALL : 0), + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; @@ -3258,7 +3153,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) * continue to assume that the obj remained out of the CPU cached * domain. */ - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; @@ -3277,21 +3172,19 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ - BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); + GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); obj->base.read_domains |= I915_GEM_DOMAIN_GTT; if (write) { obj->base.read_domains = I915_GEM_DOMAIN_GTT; obj->base.write_domain = I915_GEM_DOMAIN_GTT; - obj->dirty = 1; + obj->mm.dirty = true; } trace_i915_gem_object_change_domain(obj, old_read_domains, old_write_domain); - /* And bump the LRU for this access */ - i915_gem_object_bump_inactive_ggtt(obj); - + i915_gem_object_unpin_pages(obj); return 0; } @@ -3314,10 +3207,12 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) { struct i915_vma *vma; - int ret = 0; + int ret; + + lockdep_assert_held(&obj->base.dev->struct_mutex); if (obj->cache_level == cache_level) - goto out; + return 0; /* Inspect the list of currently bound VMA and unbind any that would * be invalid given the new cache-level. This is principally to @@ -3360,11 +3255,17 @@ restart: * If we wait upon the object, we know that all the bound * VMA are no longer active. */ - ret = i915_gem_object_wait_rendering(obj, false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; - if (!HAS_LLC(obj->base.dev) && cache_level != I915_CACHE_NONE) { + if (!HAS_LLC(to_i915(obj->base.dev)) && + cache_level != I915_CACHE_NONE) { /* Access to snoopable pages through the GTT is * incoherent and on some machines causes a hard * lockup. Relinquish the CPU mmaping to force @@ -3406,20 +3307,14 @@ restart: } } + if (obj->base.write_domain == I915_GEM_DOMAIN_CPU && + cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) + obj->cache_dirty = true; + list_for_each_entry(vma, &obj->vma_list, obj_link) vma->node.color = cache_level; obj->cache_level = cache_level; -out: - /* Flush the dirty CPU caches to the backing storage so that the - * object is now coherent at its new cache level (with respect - * to the access domain). - */ - if (obj->cache_dirty && cpu_write_needs_clflush(obj)) { - if (i915_gem_clflush_object(obj, true)) - i915_gem_chipset_flush(to_i915(obj->base.dev)); - } - return 0; } @@ -3428,10 +3323,14 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; + int err = 0; - obj = i915_gem_object_lookup(file, args->handle); - if (!obj) - return -ENOENT; + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); + if (!obj) { + err = -ENOENT; + goto out; + } switch (obj->cache_level) { case I915_CACHE_LLC: @@ -3447,15 +3346,15 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, args->caching = I915_CACHING_NONE; break; } - - i915_gem_object_put_unlocked(obj); - return 0; +out: + rcu_read_unlock(); + return err; } int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; enum i915_cache_level level; @@ -3472,23 +3371,21 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, * cacheline, whereas normally such cachelines would get * invalidated. */ - if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) + if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) return -ENODEV; level = I915_CACHE_LLC; break; case I915_CACHING_DISPLAY: - level = HAS_WT(dev_priv) ? I915_CACHE_WT : I915_CACHE_NONE; + level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; break; default: return -EINVAL; } - intel_runtime_pm_get(dev_priv); - ret = i915_mutex_lock_interruptible(dev); if (ret) - goto rpm_put; + return ret; obj = i915_gem_object_lookup(file, args->handle); if (!obj) { @@ -3497,13 +3394,9 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, } ret = i915_gem_object_set_cache_level(obj, level); - i915_gem_object_put(obj); unlock: mutex_unlock(&dev->struct_mutex); -rpm_put: - intel_runtime_pm_put(dev_priv); - return ret; } @@ -3521,6 +3414,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 old_read_domains, old_write_domain; int ret; + lockdep_assert_held(&obj->base.dev->struct_mutex); + /* Mark the pin_display early so that we account for the * display coherency whilst setting up the cache domains. */ @@ -3554,14 +3449,32 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, if (view->type == I915_GGTT_VIEW_NORMAL) vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, PIN_MAPPABLE | PIN_NONBLOCK); - if (IS_ERR(vma)) - vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, 0); + if (IS_ERR(vma)) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); + unsigned int flags; + + /* Valleyview is definitely limited to scanning out the first + * 512MiB. Lets presume this behaviour was inherited from the + * g4x display engine and that all earlier gen are similarly + * limited. Testing suggests that it is a little more + * complicated than this. For example, Cherryview appears quite + * happy to scanout from anywhere within its global aperture. + */ + flags = 0; + if (HAS_GMCH_DISPLAY(i915)) + flags = PIN_MAPPABLE; + vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags); + } if (IS_ERR(vma)) goto err_unpin_display; vma->display_alignment = max_t(u64, vma->display_alignment, alignment); - i915_gem_object_flush_cpu_write_domain(obj); + /* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */ + if (obj->cache_dirty) { + i915_gem_clflush_object(obj, true); + intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB); + } old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; @@ -3586,6 +3499,8 @@ err_unpin_display: void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) { + lockdep_assert_held(&vma->vm->dev->struct_mutex); + if (WARN_ON(vma->obj->pin_display == 0)) return; @@ -3613,7 +3528,14 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) uint32_t old_write_domain, old_read_domains; int ret; - ret = i915_gem_object_wait_rendering(obj, !write); + lockdep_assert_held(&obj->base.dev->struct_mutex); + + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + (write ? I915_WAIT_ALL : 0), + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; @@ -3635,7 +3557,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ - BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); + GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); /* If we're writing through the CPU, then the GPU read domains will * need to be invalidated at next use. @@ -3669,11 +3591,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) struct drm_i915_file_private *file_priv = file->driver_priv; unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; struct drm_i915_gem_request *request, *target = NULL; - int ret; - - ret = i915_gem_wait_for_error(&dev_priv->gpu_error); - if (ret) - return ret; + long ret; /* ABI: return -EIO if already wedged */ if (i915_terminally_wedged(&dev_priv->gpu_error)) @@ -3700,103 +3618,12 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) if (target == NULL) return 0; - ret = i915_wait_request(target, I915_WAIT_INTERRUPTIBLE, NULL, NULL); + ret = i915_wait_request(target, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT); i915_gem_request_put(target); - return ret; -} - -static bool -i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) -{ - if (!drm_mm_node_allocated(&vma->node)) - return false; - - if (vma->node.size < size) - return true; - - if (alignment && vma->node.start & (alignment - 1)) - return true; - - if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) - return true; - - if (flags & PIN_OFFSET_BIAS && - vma->node.start < (flags & PIN_OFFSET_MASK)) - return true; - - if (flags & PIN_OFFSET_FIXED && - vma->node.start != (flags & PIN_OFFSET_MASK)) - return true; - - return false; -} - -void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) -{ - struct drm_i915_gem_object *obj = vma->obj; - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - bool mappable, fenceable; - u32 fence_size, fence_alignment; - - fence_size = i915_gem_get_ggtt_size(dev_priv, - vma->size, - i915_gem_object_get_tiling(obj)); - fence_alignment = i915_gem_get_ggtt_alignment(dev_priv, - vma->size, - i915_gem_object_get_tiling(obj), - true); - - fenceable = (vma->node.size == fence_size && - (vma->node.start & (fence_alignment - 1)) == 0); - - mappable = (vma->node.start + fence_size <= - dev_priv->ggtt.mappable_end); - - /* - * Explicitly disable for rotated VMA since the display does not - * need the fence and the VMA is not accessible to other users. - */ - if (mappable && fenceable && - vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED) - vma->flags |= I915_VMA_CAN_FENCE; - else - vma->flags &= ~I915_VMA_CAN_FENCE; -} - -int __i915_vma_do_pin(struct i915_vma *vma, - u64 size, u64 alignment, u64 flags) -{ - unsigned int bound = vma->flags; - int ret; - - GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); - GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); - - if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) { - ret = -EBUSY; - goto err; - } - - if ((bound & I915_VMA_BIND_MASK) == 0) { - ret = i915_vma_insert(vma, size, alignment, flags); - if (ret) - goto err; - } - - ret = i915_vma_bind(vma, vma->obj->cache_level, flags); - if (ret) - goto err; - - if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND) - __i915_vma_set_map_and_fenceable(vma); - - GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); - return 0; - -err: - __i915_vma_unpin(vma); - return ret; + return ret < 0 ? ret : 0; } struct i915_vma * @@ -3811,6 +3638,8 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, struct i915_vma *vma; int ret; + lockdep_assert_held(&obj->base.dev->struct_mutex); + vma = i915_gem_obj_lookup_or_create_vma(obj, vm, view); if (IS_ERR(vma)) return vma; @@ -3901,83 +3730,42 @@ static __always_inline unsigned int __busy_write_id(unsigned int id) } static __always_inline unsigned int -__busy_set_if_active(const struct i915_gem_active *active, +__busy_set_if_active(const struct dma_fence *fence, unsigned int (*flag)(unsigned int id)) { - struct drm_i915_gem_request *request; + struct drm_i915_gem_request *rq; - request = rcu_dereference(active->request); - if (!request || i915_gem_request_completed(request)) - return 0; - - /* This is racy. See __i915_gem_active_get_rcu() for an in detail - * discussion of how to handle the race correctly, but for reporting - * the busy state we err on the side of potentially reporting the - * wrong engine as being busy (but we guarantee that the result - * is at least self-consistent). - * - * As we use SLAB_DESTROY_BY_RCU, the request may be reallocated - * whilst we are inspecting it, even under the RCU read lock as we are. - * This means that there is a small window for the engine and/or the - * seqno to have been overwritten. The seqno will always be in the - * future compared to the intended, and so we know that if that - * seqno is idle (on whatever engine) our request is idle and the - * return 0 above is correct. - * - * The issue is that if the engine is switched, it is just as likely - * to report that it is busy (but since the switch happened, we know - * the request should be idle). So there is a small chance that a busy - * result is actually the wrong engine. - * - * So why don't we care? - * - * For starters, the busy ioctl is a heuristic that is by definition - * racy. Even with perfect serialisation in the driver, the hardware - * state is constantly advancing - the state we report to the user - * is stale. - * - * The critical information for the busy-ioctl is whether the object - * is idle as userspace relies on that to detect whether its next - * access will stall, or if it has missed submitting commands to - * the hardware allowing the GPU to stall. We never generate a - * false-positive for idleness, thus busy-ioctl is reliable at the - * most fundamental level, and we maintain the guarantee that a - * busy object left to itself will eventually become idle (and stay - * idle!). - * - * We allow ourselves the leeway of potentially misreporting the busy - * state because that is an optimisation heuristic that is constantly - * in flux. Being quickly able to detect the busy/idle state is much - * more important than accurate logging of exactly which engines were - * busy. - * - * For accuracy in reporting the engine, we could use + /* We have to check the current hw status of the fence as the uABI + * guarantees forward progress. We could rely on the idle worker + * to eventually flush us, but to minimise latency just ask the + * hardware. * - * result = 0; - * request = __i915_gem_active_get_rcu(active); - * if (request) { - * if (!i915_gem_request_completed(request)) - * result = flag(request->engine->exec_id); - * i915_gem_request_put(request); - * } - * - * but that still remains susceptible to both hardware and userspace - * races. So we accept making the result of that race slightly worse, - * given the rarity of the race and its low impact on the result. + * Note we only report on the status of native fences. */ - return flag(READ_ONCE(request->engine->exec_id)); + if (!dma_fence_is_i915(fence)) + return 0; + + /* opencode to_request() in order to avoid const warnings */ + rq = container_of(fence, struct drm_i915_gem_request, fence); + if (i915_gem_request_completed(rq)) + return 0; + + return flag(rq->engine->exec_id); } static __always_inline unsigned int -busy_check_reader(const struct i915_gem_active *active) +busy_check_reader(const struct dma_fence *fence) { - return __busy_set_if_active(active, __busy_read_flag); + return __busy_set_if_active(fence, __busy_read_flag); } static __always_inline unsigned int -busy_check_writer(const struct i915_gem_active *active) +busy_check_writer(const struct dma_fence *fence) { - return __busy_set_if_active(active, __busy_write_id); + if (!fence) + return 0; + + return __busy_set_if_active(fence, __busy_write_id); } int @@ -3986,64 +3774,58 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_busy *args = data; struct drm_i915_gem_object *obj; - unsigned long active; + struct reservation_object_list *list; + unsigned int seq; + int err; - obj = i915_gem_object_lookup(file, args->handle); + err = -ENOENT; + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); if (!obj) - return -ENOENT; + goto out; - args->busy = 0; - active = __I915_BO_ACTIVE(obj); - if (active) { - int idx; + /* A discrepancy here is that we do not report the status of + * non-i915 fences, i.e. even though we may report the object as idle, + * a call to set-domain may still stall waiting for foreign rendering. + * This also means that wait-ioctl may report an object as busy, + * where busy-ioctl considers it idle. + * + * We trade the ability to warn of foreign fences to report on which + * i915 engines are active for the object. + * + * Alternatively, we can trade that extra information on read/write + * activity with + * args->busy = + * !reservation_object_test_signaled_rcu(obj->resv, true); + * to report the overall busyness. This is what the wait-ioctl does. + * + */ +retry: + seq = raw_read_seqcount(&obj->resv->seq); - /* Yes, the lookups are intentionally racy. - * - * First, we cannot simply rely on __I915_BO_ACTIVE. We have - * to regard the value as stale and as our ABI guarantees - * forward progress, we confirm the status of each active - * request with the hardware. - * - * Even though we guard the pointer lookup by RCU, that only - * guarantees that the pointer and its contents remain - * dereferencable and does *not* mean that the request we - * have is the same as the one being tracked by the object. - * - * Consider that we lookup the request just as it is being - * retired and freed. We take a local copy of the pointer, - * but before we add its engine into the busy set, the other - * thread reallocates it and assigns it to a task on another - * engine with a fresh and incomplete seqno. Guarding against - * that requires careful serialisation and reference counting, - * i.e. using __i915_gem_active_get_request_rcu(). We don't, - * instead we expect that if the result is busy, which engines - * are busy is not completely reliable - we only guarantee - * that the object was busy. - */ - rcu_read_lock(); + /* Translate the exclusive fence to the READ *and* WRITE engine */ + args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl)); - for_each_active(active, idx) - args->busy |= busy_check_reader(&obj->last_read[idx]); + /* Translate shared fences to READ set of engines */ + list = rcu_dereference(obj->resv->fence); + if (list) { + unsigned int shared_count = list->shared_count, i; - /* For ABI sanity, we only care that the write engine is in - * the set of read engines. This should be ensured by the - * ordering of setting last_read/last_write in - * i915_vma_move_to_active(), and then in reverse in retire. - * However, for good measure, we always report the last_write - * request as a busy read as well as being a busy write. - * - * We don't care that the set of active read/write engines - * may change during construction of the result, as it is - * equally liable to change before userspace can inspect - * the result. - */ - args->busy |= busy_check_writer(&obj->last_write); + for (i = 0; i < shared_count; ++i) { + struct dma_fence *fence = + rcu_dereference(list->shared[i]); - rcu_read_unlock(); + args->busy |= busy_check_reader(fence); + } } - i915_gem_object_put_unlocked(obj); - return 0; + if (args->busy && read_seqcount_retry(&obj->resv->seq, seq)) + goto retry; + + err = 0; +out: + rcu_read_unlock(); + return err; } int @@ -4060,7 +3842,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_madvise *args = data; struct drm_i915_gem_object *obj; - int ret; + int err; switch (args->madv) { case I915_MADV_DONTNEED: @@ -4070,65 +3852,83 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, return -EINVAL; } - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ret; - obj = i915_gem_object_lookup(file_priv, args->handle); - if (!obj) { - ret = -ENOENT; - goto unlock; - } + if (!obj) + return -ENOENT; + + err = mutex_lock_interruptible(&obj->mm.lock); + if (err) + goto out; - if (obj->pages && + if (obj->mm.pages && i915_gem_object_is_tiled(obj) && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { - if (obj->madv == I915_MADV_WILLNEED) - i915_gem_object_unpin_pages(obj); - if (args->madv == I915_MADV_WILLNEED) - i915_gem_object_pin_pages(obj); + if (obj->mm.madv == I915_MADV_WILLNEED) { + GEM_BUG_ON(!obj->mm.quirked); + __i915_gem_object_unpin_pages(obj); + obj->mm.quirked = false; + } + if (args->madv == I915_MADV_WILLNEED) { + GEM_BUG_ON(obj->mm.quirked); + __i915_gem_object_pin_pages(obj); + obj->mm.quirked = true; + } } - if (obj->madv != __I915_MADV_PURGED) - obj->madv = args->madv; + if (obj->mm.madv != __I915_MADV_PURGED) + obj->mm.madv = args->madv; /* if the object is no longer attached, discard its backing storage */ - if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) + if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages) i915_gem_object_truncate(obj); - args->retained = obj->madv != __I915_MADV_PURGED; + args->retained = obj->mm.madv != __I915_MADV_PURGED; + mutex_unlock(&obj->mm.lock); +out: i915_gem_object_put(obj); -unlock: - mutex_unlock(&dev->struct_mutex); - return ret; + return err; +} + +static void +frontbuffer_retire(struct i915_gem_active *active, + struct drm_i915_gem_request *request) +{ + struct drm_i915_gem_object *obj = + container_of(active, typeof(*obj), frontbuffer_write); + + intel_fb_obj_flush(obj, true, ORIGIN_CS); } void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops) { - int i; + mutex_init(&obj->mm.lock); - INIT_LIST_HEAD(&obj->global_list); - for (i = 0; i < I915_NUM_ENGINES; i++) - init_request_active(&obj->last_read[i], - i915_gem_object_retire__read); - init_request_active(&obj->last_write, - i915_gem_object_retire__write); + INIT_LIST_HEAD(&obj->global_link); + INIT_LIST_HEAD(&obj->userfault_link); INIT_LIST_HEAD(&obj->obj_exec_link); INIT_LIST_HEAD(&obj->vma_list); INIT_LIST_HEAD(&obj->batch_pool_link); obj->ops = ops; + reservation_object_init(&obj->__builtin_resv); + obj->resv = &obj->__builtin_resv; + obj->frontbuffer_ggtt_origin = ORIGIN_GTT; - obj->madv = I915_MADV_WILLNEED; + init_request_active(&obj->frontbuffer_write, frontbuffer_retire); + + obj->mm.madv = I915_MADV_WILLNEED; + INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); + mutex_init(&obj->mm.get_page.lock); i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size); } static const struct drm_i915_gem_object_ops i915_gem_object_ops = { - .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, + .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = i915_gem_object_get_pages_gtt, .put_pages = i915_gem_object_put_pages_gtt, }; @@ -4140,6 +3940,7 @@ static const struct drm_i915_gem_object_ops i915_gem_object_ops = { struct drm_i915_gem_object * i915_gem_object_create(struct drm_device *dev, u64 size) { + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; struct address_space *mapping; gfp_t mask; @@ -4165,7 +3966,7 @@ i915_gem_object_create(struct drm_device *dev, u64 size) goto fail; mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; - if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { + if (IS_CRESTLINE(dev_priv) || IS_BROADWATER(dev_priv)) { /* 965gm cannot relocate objects above 4GiB. */ mask &= ~__GFP_HIGHMEM; mask |= __GFP_DMA32; @@ -4179,7 +3980,7 @@ i915_gem_object_create(struct drm_device *dev, u64 size) obj->base.write_domain = I915_GEM_DOMAIN_CPU; obj->base.read_domains = I915_GEM_DOMAIN_CPU; - if (HAS_LLC(dev)) { + if (HAS_LLC(dev_priv)) { /* On some devices, we can have the GPU use the LLC (the CPU * cache) for about a 10% performance improvement * compared to uncached. Graphics requests other than @@ -4202,7 +4003,6 @@ i915_gem_object_create(struct drm_device *dev, u64 size) fail: i915_gem_object_free(obj); - return ERR_PTR(ret); } @@ -4214,7 +4014,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj) * back the contents from the GPU. */ - if (obj->madv != I915_MADV_WILLNEED) + if (obj->mm.madv != I915_MADV_WILLNEED) return false; if (obj->base.filp == NULL) @@ -4230,16 +4030,72 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj) return atomic_long_read(&obj->base.filp->f_count) == 1; } -void i915_gem_free_object(struct drm_gem_object *gem_obj) +static void __i915_gem_free_objects(struct drm_i915_private *i915, + struct llist_node *freed) { - struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_vma *vma, *next; + struct drm_i915_gem_object *obj, *on; - intel_runtime_pm_get(dev_priv); + mutex_lock(&i915->drm.struct_mutex); + intel_runtime_pm_get(i915); + llist_for_each_entry(obj, freed, freed) { + struct i915_vma *vma, *vn; + + trace_i915_gem_object_destroy(obj); + + GEM_BUG_ON(i915_gem_object_is_active(obj)); + list_for_each_entry_safe(vma, vn, + &obj->vma_list, obj_link) { + GEM_BUG_ON(!i915_vma_is_ggtt(vma)); + GEM_BUG_ON(i915_vma_is_active(vma)); + vma->flags &= ~I915_VMA_PIN_MASK; + i915_vma_close(vma); + } + GEM_BUG_ON(!list_empty(&obj->vma_list)); + GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree)); + + list_del(&obj->global_link); + } + intel_runtime_pm_put(i915); + mutex_unlock(&i915->drm.struct_mutex); + + llist_for_each_entry_safe(obj, on, freed, freed) { + GEM_BUG_ON(obj->bind_count); + GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits)); + + if (obj->ops->release) + obj->ops->release(obj); - trace_i915_gem_object_destroy(obj); + if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) + atomic_set(&obj->mm.pages_pin_count, 0); + __i915_gem_object_put_pages(obj, I915_MM_NORMAL); + GEM_BUG_ON(obj->mm.pages); + + if (obj->base.import_attach) + drm_prime_gem_destroy(&obj->base, NULL); + + reservation_object_fini(&obj->__builtin_resv); + drm_gem_object_release(&obj->base); + i915_gem_info_remove_obj(i915, obj->base.size); + + kfree(obj->bit_17); + i915_gem_object_free(obj); + } +} + +static void i915_gem_flush_free_objects(struct drm_i915_private *i915) +{ + struct llist_node *freed; + + freed = llist_del_all(&i915->mm.free_list); + if (unlikely(freed)) + __i915_gem_free_objects(i915, freed); +} + +static void __i915_gem_free_work(struct work_struct *work) +{ + struct drm_i915_private *i915 = + container_of(work, struct drm_i915_private, mm.free_work); + struct llist_node *freed; /* All file-owned VMA should have been released by this point through * i915_gem_close_object(), or earlier by i915_gem_context_close(). @@ -4248,47 +4104,62 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) * the GTT either for the user or for scanout). Those VMA still need to * unbound now. */ - list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) { - GEM_BUG_ON(!i915_vma_is_ggtt(vma)); - GEM_BUG_ON(i915_vma_is_active(vma)); - vma->flags &= ~I915_VMA_PIN_MASK; - i915_vma_close(vma); - } - GEM_BUG_ON(obj->bind_count); - /* Stolen objects don't hold a ref, but do hold pin count. Fix that up - * before progressing. */ - if (obj->stolen) - i915_gem_object_unpin_pages(obj); + while ((freed = llist_del_all(&i915->mm.free_list))) + __i915_gem_free_objects(i915, freed); +} - WARN_ON(atomic_read(&obj->frontbuffer_bits)); +static void __i915_gem_free_object_rcu(struct rcu_head *head) +{ + struct drm_i915_gem_object *obj = + container_of(head, typeof(*obj), rcu); + struct drm_i915_private *i915 = to_i915(obj->base.dev); - if (obj->pages && obj->madv == I915_MADV_WILLNEED && - dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES && - i915_gem_object_is_tiled(obj)) - i915_gem_object_unpin_pages(obj); + /* We can't simply use call_rcu() from i915_gem_free_object() + * as we need to block whilst unbinding, and the call_rcu + * task may be called from softirq context. So we take a + * detour through a worker. + */ + if (llist_add(&obj->freed, &i915->mm.free_list)) + schedule_work(&i915->mm.free_work); +} - if (WARN_ON(obj->pages_pin_count)) - obj->pages_pin_count = 0; - if (discard_backing_storage(obj)) - obj->madv = I915_MADV_DONTNEED; - i915_gem_object_put_pages(obj); +void i915_gem_free_object(struct drm_gem_object *gem_obj) +{ + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - BUG_ON(obj->pages); + if (obj->mm.quirked) + __i915_gem_object_unpin_pages(obj); - if (obj->base.import_attach) - drm_prime_gem_destroy(&obj->base, NULL); + if (discard_backing_storage(obj)) + obj->mm.madv = I915_MADV_DONTNEED; - if (obj->ops->release) - obj->ops->release(obj); + /* Before we free the object, make sure any pure RCU-only + * read-side critical sections are complete, e.g. + * i915_gem_busy_ioctl(). For the corresponding synchronized + * lookup see i915_gem_object_lookup_rcu(). + */ + call_rcu(&obj->rcu, __i915_gem_free_object_rcu); +} - drm_gem_object_release(&obj->base); - i915_gem_info_remove_obj(dev_priv, obj->base.size); +void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj) +{ + lockdep_assert_held(&obj->base.dev->struct_mutex); - kfree(obj->bit_17); - i915_gem_object_free(obj); + GEM_BUG_ON(i915_gem_object_has_active_reference(obj)); + if (i915_gem_object_is_active(obj)) + i915_gem_object_set_active_reference(obj); + else + i915_gem_object_put(obj); +} - intel_runtime_pm_put(dev_priv); +static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, dev_priv, id) + GEM_BUG_ON(engine->last_context != dev_priv->kernel_context); } int i915_gem_suspend(struct drm_device *dev) @@ -4319,18 +4190,22 @@ int i915_gem_suspend(struct drm_device *dev) goto err; i915_gem_retire_requests(dev_priv); + GEM_BUG_ON(dev_priv->gt.active_requests); + assert_kernel_context_is_current(dev_priv); i915_gem_context_lost(dev_priv); mutex_unlock(&dev->struct_mutex); cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); cancel_delayed_work_sync(&dev_priv->gt.retire_work); flush_delayed_work(&dev_priv->gt.idle_work); + flush_work(&dev_priv->mm.free_work); /* Assert that we sucessfully flushed all the work and * reset the GPU back to its idle, low power state. */ WARN_ON(dev_priv->gt.awake); + WARN_ON(!intel_execlists_idle(dev_priv)); /* * Neither the BIOS, ourselves or any other kernel @@ -4351,7 +4226,7 @@ int i915_gem_suspend(struct drm_device *dev) * machines is a good idea, we don't - just in case it leaves the * machine in an unusable condition. */ - if (HAS_HW_CONTEXTS(dev)) { + if (HAS_HW_CONTEXTS(dev_priv)) { int reset = intel_gpu_reset(dev_priv, ALL_ENGINES); WARN_ON(reset && reset != -ENODEV); } @@ -4367,8 +4242,10 @@ void i915_gem_resume(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + WARN_ON(dev_priv->gt.awake); + mutex_lock(&dev->struct_mutex); - i915_gem_restore_gtt_mappings(dev); + i915_gem_restore_gtt_mappings(dev_priv); /* As we didn't flush the kernel context before suspend, we cannot * guarantee that the context image is complete. So let's just reset @@ -4379,11 +4256,9 @@ void i915_gem_resume(struct drm_device *dev) mutex_unlock(&dev->struct_mutex); } -void i915_gem_init_swizzling(struct drm_device *dev) +void i915_gem_init_swizzling(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - if (INTEL_INFO(dev)->gen < 5 || + if (INTEL_GEN(dev_priv) < 5 || dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) return; @@ -4437,10 +4312,12 @@ i915_gem_init_hw(struct drm_device *dev) enum intel_engine_id id; int ret; + dev_priv->gt.last_init_time = ktime_get(); + /* Double layer security blanket, see i915_gem_init() */ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); - if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9) + if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9) I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); if (IS_HASWELL(dev_priv)) @@ -4452,14 +4329,14 @@ i915_gem_init_hw(struct drm_device *dev) u32 temp = I915_READ(GEN7_MSG_CTL); temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); I915_WRITE(GEN7_MSG_CTL, temp); - } else if (INTEL_INFO(dev)->gen >= 7) { + } else if (INTEL_GEN(dev_priv) >= 7) { u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); temp &= ~RESET_PCH_HANDSHAKE_ENABLE; I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); } } - i915_gem_init_swizzling(dev); + i915_gem_init_swizzling(dev_priv); /* * At least 830 can leave some of the unused rings @@ -4471,7 +4348,7 @@ i915_gem_init_hw(struct drm_device *dev) BUG_ON(!dev_priv->kernel_context); - ret = i915_ppgtt_init_hw(dev); + ret = i915_ppgtt_init_hw(dev_priv); if (ret) { DRM_ERROR("PPGTT enable HW failed %d\n", ret); goto out; @@ -4586,7 +4463,6 @@ i915_gem_cleanup_engines(struct drm_device *dev) void i915_gem_load_init_fences(struct drm_i915_private *dev_priv) { - struct drm_device *dev = &dev_priv->drm; int i; if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) && @@ -4610,38 +4486,52 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv) fence->id = i; list_add_tail(&fence->link, &dev_priv->mm.fence_list); } - i915_gem_restore_fences(dev); + i915_gem_restore_fences(dev_priv); - i915_gem_detect_bit_6_swizzle(dev); + i915_gem_detect_bit_6_swizzle(dev_priv); } -void +int i915_gem_load_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + int err = -ENOMEM; + + dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); + if (!dev_priv->objects) + goto err_out; - dev_priv->objects = - kmem_cache_create("i915_gem_object", - sizeof(struct drm_i915_gem_object), 0, - SLAB_HWCACHE_ALIGN, - NULL); - dev_priv->vmas = - kmem_cache_create("i915_gem_vma", - sizeof(struct i915_vma), 0, - SLAB_HWCACHE_ALIGN, - NULL); - dev_priv->requests = - kmem_cache_create("i915_gem_request", - sizeof(struct drm_i915_gem_request), 0, - SLAB_HWCACHE_ALIGN | - SLAB_RECLAIM_ACCOUNT | - SLAB_DESTROY_BY_RCU, - NULL); + dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); + if (!dev_priv->vmas) + goto err_objects; + + dev_priv->requests = KMEM_CACHE(drm_i915_gem_request, + SLAB_HWCACHE_ALIGN | + SLAB_RECLAIM_ACCOUNT | + SLAB_DESTROY_BY_RCU); + if (!dev_priv->requests) + goto err_vmas; + + dev_priv->dependencies = KMEM_CACHE(i915_dependency, + SLAB_HWCACHE_ALIGN | + SLAB_RECLAIM_ACCOUNT); + if (!dev_priv->dependencies) + goto err_requests; + + mutex_lock(&dev_priv->drm.struct_mutex); + INIT_LIST_HEAD(&dev_priv->gt.timelines); + err = i915_gem_timeline_init__global(dev_priv); + mutex_unlock(&dev_priv->drm.struct_mutex); + if (err) + goto err_dependencies; INIT_LIST_HEAD(&dev_priv->context_list); + INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work); + init_llist_head(&dev_priv->mm.free_list); INIT_LIST_HEAD(&dev_priv->mm.unbound_list); INIT_LIST_HEAD(&dev_priv->mm.bound_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); + INIT_LIST_HEAD(&dev_priv->mm.userfault_list); INIT_DELAYED_WORK(&dev_priv->gt.retire_work, i915_gem_retire_work_handler); INIT_DELAYED_WORK(&dev_priv->gt.idle_work, @@ -4658,12 +4548,33 @@ i915_gem_load_init(struct drm_device *dev) atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0); spin_lock_init(&dev_priv->fb_tracking.lock); + + return 0; + +err_dependencies: + kmem_cache_destroy(dev_priv->dependencies); +err_requests: + kmem_cache_destroy(dev_priv->requests); +err_vmas: + kmem_cache_destroy(dev_priv->vmas); +err_objects: + kmem_cache_destroy(dev_priv->objects); +err_out: + return err; } void i915_gem_load_cleanup(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + WARN_ON(!llist_empty(&dev_priv->mm.free_list)); + + mutex_lock(&dev_priv->drm.struct_mutex); + i915_gem_timeline_fini(&dev_priv->gt.global_timeline); + WARN_ON(!list_empty(&dev_priv->gt.timelines)); + mutex_unlock(&dev_priv->drm.struct_mutex); + + kmem_cache_destroy(dev_priv->dependencies); kmem_cache_destroy(dev_priv->requests); kmem_cache_destroy(dev_priv->vmas); kmem_cache_destroy(dev_priv->objects); @@ -4712,7 +4623,7 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv) i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND); for (p = phases; *p; p++) { - list_for_each_entry(obj, *p, global_list) { + list_for_each_entry(obj, *p, global_link) { obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; } @@ -4748,7 +4659,7 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file) struct drm_i915_file_private *file_priv; int ret; - DRM_DEBUG_DRIVER("\n"); + DRM_DEBUG("\n"); file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); if (!file_priv) @@ -4804,21 +4715,6 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, } } -/* Like i915_gem_object_get_page(), but mark the returned page dirty */ -struct page * -i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n) -{ - struct page *page; - - /* Only default objects have per-page dirty tracking */ - if (WARN_ON(!i915_gem_object_has_struct_page(obj))) - return NULL; - - page = i915_gem_object_get_page(obj, n); - set_page_dirty(page); - return page; -} - /* Allocate a new GEM object and fill it with the supplied data */ struct drm_i915_gem_object * i915_gem_object_create_from_data(struct drm_device *dev, @@ -4837,14 +4733,13 @@ i915_gem_object_create_from_data(struct drm_device *dev, if (ret) goto fail; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) goto fail; - i915_gem_object_pin_pages(obj); - sg = obj->pages; + sg = obj->mm.pages; bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size); - obj->dirty = 1; /* Backing store is now out of date */ + obj->mm.dirty = true; /* Backing store is now out of date */ i915_gem_object_unpin_pages(obj); if (WARN_ON(bytes != size)) { @@ -4859,3 +4754,156 @@ fail: i915_gem_object_put(obj); return ERR_PTR(ret); } + +struct scatterlist * +i915_gem_object_get_sg(struct drm_i915_gem_object *obj, + unsigned int n, + unsigned int *offset) +{ + struct i915_gem_object_page_iter *iter = &obj->mm.get_page; + struct scatterlist *sg; + unsigned int idx, count; + + might_sleep(); + GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + + /* As we iterate forward through the sg, we record each entry in a + * radixtree for quick repeated (backwards) lookups. If we have seen + * this index previously, we will have an entry for it. + * + * Initial lookup is O(N), but this is amortized to O(1) for + * sequential page access (where each new request is consecutive + * to the previous one). Repeated lookups are O(lg(obj->base.size)), + * i.e. O(1) with a large constant! + */ + if (n < READ_ONCE(iter->sg_idx)) + goto lookup; + + mutex_lock(&iter->lock); + + /* We prefer to reuse the last sg so that repeated lookup of this + * (or the subsequent) sg are fast - comparing against the last + * sg is faster than going through the radixtree. + */ + + sg = iter->sg_pos; + idx = iter->sg_idx; + count = __sg_page_count(sg); + + while (idx + count <= n) { + unsigned long exception, i; + int ret; + + /* If we cannot allocate and insert this entry, or the + * individual pages from this range, cancel updating the + * sg_idx so that on this lookup we are forced to linearly + * scan onwards, but on future lookups we will try the + * insertion again (in which case we need to be careful of + * the error return reporting that we have already inserted + * this index). + */ + ret = radix_tree_insert(&iter->radix, idx, sg); + if (ret && ret != -EEXIST) + goto scan; + + exception = + RADIX_TREE_EXCEPTIONAL_ENTRY | + idx << RADIX_TREE_EXCEPTIONAL_SHIFT; + for (i = 1; i < count; i++) { + ret = radix_tree_insert(&iter->radix, idx + i, + (void *)exception); + if (ret && ret != -EEXIST) + goto scan; + } + + idx += count; + sg = ____sg_next(sg); + count = __sg_page_count(sg); + } + +scan: + iter->sg_pos = sg; + iter->sg_idx = idx; + + mutex_unlock(&iter->lock); + + if (unlikely(n < idx)) /* insertion completed by another thread */ + goto lookup; + + /* In case we failed to insert the entry into the radixtree, we need + * to look beyond the current sg. + */ + while (idx + count <= n) { + idx += count; + sg = ____sg_next(sg); + count = __sg_page_count(sg); + } + + *offset = n - idx; + return sg; + +lookup: + rcu_read_lock(); + + sg = radix_tree_lookup(&iter->radix, n); + GEM_BUG_ON(!sg); + + /* If this index is in the middle of multi-page sg entry, + * the radixtree will contain an exceptional entry that points + * to the start of that range. We will return the pointer to + * the base page and the offset of this page within the + * sg entry's range. + */ + *offset = 0; + if (unlikely(radix_tree_exception(sg))) { + unsigned long base = + (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT; + + sg = radix_tree_lookup(&iter->radix, base); + GEM_BUG_ON(!sg); + + *offset = n - base; + } + + rcu_read_unlock(); + + return sg; +} + +struct page * +i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) +{ + struct scatterlist *sg; + unsigned int offset; + + GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); + + sg = i915_gem_object_get_sg(obj, n, &offset); + return nth_page(sg_page(sg), offset); +} + +/* Like i915_gem_object_get_page(), but mark the returned page dirty */ +struct page * +i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, + unsigned int n) +{ + struct page *page; + + page = i915_gem_object_get_page(obj, n); + if (!obj->mm.dirty) + set_page_dirty(page); + + return page; +} + +dma_addr_t +i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, + unsigned long n) +{ + struct scatterlist *sg; + unsigned int offset; + + sg = i915_gem_object_get_sg(obj, n, &offset); + return sg_dma_address(sg) + (offset << PAGE_SHIFT); +} diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index 8292e797d9b5..51ec793f2e20 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -28,7 +28,9 @@ #ifdef CONFIG_DRM_I915_DEBUG_GEM #define GEM_BUG_ON(expr) BUG_ON(expr) #else -#define GEM_BUG_ON(expr) +#define GEM_BUG_ON(expr) do { } while (0) #endif +#define I915_NUM_ENGINES 5 + #endif /* __I915_GEM_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c index ed989596d9a3..b3bc119ec1bb 100644 --- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c +++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c @@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool) list_for_each_entry_safe(obj, next, &pool->cache_list[n], batch_pool_link) - i915_gem_object_put(obj); + __i915_gem_object_release_unless_active(obj); INIT_LIST_HEAD(&pool->cache_list[n]); } @@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) { struct drm_i915_gem_object *obj = NULL; - struct drm_i915_gem_object *tmp, *next; + struct drm_i915_gem_object *tmp; struct list_head *list; - int n; + int n, ret; lockdep_assert_held(&pool->engine->i915->drm.struct_mutex); @@ -112,40 +112,35 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n = ARRAY_SIZE(pool->cache_list) - 1; list = &pool->cache_list[n]; - list_for_each_entry_safe(tmp, next, list, batch_pool_link) { + list_for_each_entry(tmp, list, batch_pool_link) { /* The batches are strictly LRU ordered */ - if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id], - &tmp->base.dev->struct_mutex)) + if (i915_gem_object_is_active(tmp)) break; - /* While we're looping, do some clean up */ - if (tmp->madv == __I915_MADV_PURGED) { - list_del(&tmp->batch_pool_link); - i915_gem_object_put(tmp); - continue; - } + GEM_BUG_ON(!reservation_object_test_signaled_rcu(tmp->resv, + true)); if (tmp->base.size >= size) { + /* Clear the set of shared fences early */ + ww_mutex_lock(&tmp->resv->lock, NULL); + reservation_object_add_excl_fence(tmp->resv, NULL); + ww_mutex_unlock(&tmp->resv->lock); + obj = tmp; break; } } if (obj == NULL) { - int ret; - - obj = i915_gem_object_create(&pool->engine->i915->drm, size); + obj = i915_gem_object_create_internal(pool->engine->i915, size); if (IS_ERR(obj)) return obj; - - ret = i915_gem_object_get_pages(obj); - if (ret) - return ERR_PTR(ret); - - obj->madv = I915_MADV_DONTNEED; } + ret = i915_gem_object_pin_pages(obj); + if (ret) + return ERR_PTR(ret); + list_move_tail(&obj->batch_pool_link, list); - i915_gem_object_pin_pages(obj); return obj; } diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 5dca32ac1c67..1f94b8d6d83d 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -155,9 +155,10 @@ void i915_gem_context_free(struct kref *ctx_ref) if (ce->ring) intel_ring_free(ce->ring); - i915_vma_put(ce->state); + __i915_gem_object_release_unless_active(ce->state->obj); } + kfree(ctx->name); put_pid(ctx->pid); list_del(&ctx->link); @@ -303,19 +304,28 @@ __create_hw_context(struct drm_device *dev, } /* Default context will never have a file_priv */ - if (file_priv != NULL) { + ret = DEFAULT_CONTEXT_HANDLE; + if (file_priv) { ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL); if (ret < 0) goto err_out; - } else - ret = DEFAULT_CONTEXT_HANDLE; + } + ctx->user_handle = ret; ctx->file_priv = file_priv; - if (file_priv) + if (file_priv) { ctx->pid = get_task_pid(current, PIDTYPE_PID); + ctx->name = kasprintf(GFP_KERNEL, "%s[%d]/%x", + current->comm, + pid_nr(ctx->pid), + ctx->user_handle); + if (!ctx->name) { + ret = -ENOMEM; + goto err_pid; + } + } - ctx->user_handle = ret; /* NB: Mark all slices as needing a remap so that when the context first * loads it will restore whatever remap state already exists. If there * is no remap info, it will be a NOP. */ @@ -329,6 +339,9 @@ __create_hw_context(struct drm_device *dev, return ctx; +err_pid: + put_pid(ctx->pid); + idr_remove(&file_priv->context_idr, ctx->user_handle); err_out: context_close(ctx); return ERR_PTR(ret); @@ -352,9 +365,9 @@ i915_gem_create_context(struct drm_device *dev, return ctx; if (USES_FULL_PPGTT(dev)) { - struct i915_hw_ppgtt *ppgtt = - i915_ppgtt_create(to_i915(dev), file_priv); + struct i915_hw_ppgtt *ppgtt; + ppgtt = i915_ppgtt_create(to_i915(dev), file_priv, ctx->name); if (IS_ERR(ppgtt)) { DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", PTR_ERR(ppgtt)); @@ -463,6 +476,7 @@ int i915_gem_context_init(struct drm_device *dev) return PTR_ERR(ctx); } + ctx->priority = I915_PRIORITY_MIN; /* lowest priority; idle task */ dev_priv->kernel_context = ctx; DRM_DEBUG_DRIVER("%s context support initialized\n", @@ -751,12 +765,36 @@ needs_pd_load_post(struct i915_hw_ppgtt *ppgtt, return false; } +struct i915_vma * +i915_gem_context_pin_legacy(struct i915_gem_context *ctx, + unsigned int flags) +{ + struct i915_vma *vma = ctx->engine[RCS].state; + int ret; + + /* Clear this page out of any CPU caches for coherent swap-in/out. + * We only want to do this on the first bind so that we do not stall + * on an active context (which by nature is already on the GPU). + */ + if (!(vma->flags & I915_VMA_GLOBAL_BIND)) { + ret = i915_gem_object_set_to_gtt_domain(vma->obj, false); + if (ret) + return ERR_PTR(ret); + } + + ret = i915_vma_pin(vma, 0, ctx->ggtt_alignment, PIN_GLOBAL | flags); + if (ret) + return ERR_PTR(ret); + + return vma; +} + static int do_rcs_switch(struct drm_i915_gem_request *req) { struct i915_gem_context *to = req->ctx; struct intel_engine_cs *engine = req->engine; struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt; - struct i915_vma *vma = to->engine[RCS].state; + struct i915_vma *vma; struct i915_gem_context *from; u32 hw_flags; int ret, i; @@ -764,17 +802,10 @@ static int do_rcs_switch(struct drm_i915_gem_request *req) if (skip_rcs_switch(ppgtt, engine, to)) return 0; - /* Clear this page out of any CPU caches for coherent swap-in/out. */ - if (!(vma->flags & I915_VMA_GLOBAL_BIND)) { - ret = i915_gem_object_set_to_gtt_domain(vma->obj, false); - if (ret) - return ret; - } - /* Trying to pin first makes error handling easier. */ - ret = i915_vma_pin(vma, 0, to->ggtt_alignment, PIN_GLOBAL); - if (ret) - return ret; + vma = i915_gem_context_pin_legacy(to, 0); + if (IS_ERR(vma)) + return PTR_ERR(vma); /* * Pin can switch back to the default context if we end up calling into @@ -931,22 +962,33 @@ int i915_switch_context(struct drm_i915_gem_request *req) int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv) { struct intel_engine_cs *engine; + struct i915_gem_timeline *timeline; enum intel_engine_id id; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + for_each_engine(engine, dev_priv, id) { struct drm_i915_gem_request *req; int ret; - if (engine->last_context == NULL) - continue; - - if (engine->last_context == dev_priv->kernel_context) - continue; - req = i915_gem_request_alloc(engine, dev_priv->kernel_context); if (IS_ERR(req)) return PTR_ERR(req); + /* Queue this switch after all other activity */ + list_for_each_entry(timeline, &dev_priv->gt.timelines, link) { + struct drm_i915_gem_request *prev; + struct intel_timeline *tl; + + tl = &timeline->engine[engine->id]; + prev = i915_gem_active_raw(&tl->last_request, + &dev_priv->drm.struct_mutex); + if (prev) + i915_sw_fence_await_sw_fence_gfp(&req->submit, + &prev->submit, + GFP_KERNEL); + } + ret = i915_switch_context(req); i915_add_request_no_flush(req); if (ret) diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c index 97c9d68b45df..5e38299b5df6 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c @@ -44,51 +44,42 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme struct scatterlist *src, *dst; int ret, i; - ret = i915_mutex_lock_interruptible(obj->base.dev); + ret = i915_gem_object_pin_pages(obj); if (ret) goto err; - ret = i915_gem_object_get_pages(obj); - if (ret) - goto err_unlock; - - i915_gem_object_pin_pages(obj); - /* Copy sg so that we make an independent mapping */ st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (st == NULL) { ret = -ENOMEM; - goto err_unpin; + goto err_unpin_pages; } - ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL); + ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL); if (ret) goto err_free; - src = obj->pages->sgl; + src = obj->mm.pages->sgl; dst = st->sgl; - for (i = 0; i < obj->pages->nents; i++) { + for (i = 0; i < obj->mm.pages->nents; i++) { sg_set_page(dst, sg_page(src), src->length, 0); dst = sg_next(dst); src = sg_next(src); } if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) { - ret =-ENOMEM; + ret = -ENOMEM; goto err_free_sg; } - mutex_unlock(&obj->base.dev->struct_mutex); return st; err_free_sg: sg_free_table(st); err_free: kfree(st); -err_unpin: +err_unpin_pages: i915_gem_object_unpin_pages(obj); -err_unlock: - mutex_unlock(&obj->base.dev->struct_mutex); err: return ERR_PTR(ret); } @@ -103,36 +94,21 @@ static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, sg_free_table(sg); kfree(sg); - mutex_lock(&obj->base.dev->struct_mutex); i915_gem_object_unpin_pages(obj); - mutex_unlock(&obj->base.dev->struct_mutex); } static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); - struct drm_device *dev = obj->base.dev; - void *addr; - int ret; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ERR_PTR(ret); - - addr = i915_gem_object_pin_map(obj, I915_MAP_WB); - mutex_unlock(&dev->struct_mutex); - - return addr; + return i915_gem_object_pin_map(obj, I915_MAP_WB); } static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); - struct drm_device *dev = obj->base.dev; - mutex_lock(&dev->struct_mutex); i915_gem_object_unpin_map(obj); - mutex_unlock(&dev->struct_mutex); } static void *i915_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf, unsigned long page_num) @@ -179,32 +155,45 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); struct drm_device *dev = obj->base.dev; - int ret; bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE); + int err; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ret; + err = i915_gem_object_pin_pages(obj); + if (err) + return err; + + err = i915_mutex_lock_interruptible(dev); + if (err) + goto out; - ret = i915_gem_object_set_to_cpu_domain(obj, write); + err = i915_gem_object_set_to_cpu_domain(obj, write); mutex_unlock(&dev->struct_mutex); - return ret; + +out: + i915_gem_object_unpin_pages(obj); + return err; } static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); struct drm_device *dev = obj->base.dev; - int ret; + int err; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ret; + err = i915_gem_object_pin_pages(obj); + if (err) + return err; - ret = i915_gem_object_set_to_gtt_domain(obj, false); + err = i915_mutex_lock_interruptible(dev); + if (err) + goto out; + + err = i915_gem_object_set_to_gtt_domain(obj, false); mutex_unlock(&dev->struct_mutex); - return ret; +out: + i915_gem_object_unpin_pages(obj); + return err; } static const struct dma_buf_ops i915_dmabuf_ops = { @@ -222,60 +211,17 @@ static const struct dma_buf_ops i915_dmabuf_ops = { .end_cpu_access = i915_gem_end_cpu_access, }; -static void export_fences(struct drm_i915_gem_object *obj, - struct dma_buf *dma_buf) -{ - struct reservation_object *resv = dma_buf->resv; - struct drm_i915_gem_request *req; - unsigned long active; - int idx; - - active = __I915_BO_ACTIVE(obj); - if (!active) - return; - - /* Serialise with execbuf to prevent concurrent fence-loops */ - mutex_lock(&obj->base.dev->struct_mutex); - - /* Mark the object for future fences before racily adding old fences */ - obj->base.dma_buf = dma_buf; - - ww_mutex_lock(&resv->lock, NULL); - - for_each_active(active, idx) { - req = i915_gem_active_get(&obj->last_read[idx], - &obj->base.dev->struct_mutex); - if (!req) - continue; - - if (reservation_object_reserve_shared(resv) == 0) - reservation_object_add_shared_fence(resv, &req->fence); - - i915_gem_request_put(req); - } - - req = i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); - if (req) { - reservation_object_add_excl_fence(resv, &req->fence); - i915_gem_request_put(req); - } - - ww_mutex_unlock(&resv->lock); - mutex_unlock(&obj->base.dev->struct_mutex); -} - struct dma_buf *i915_gem_prime_export(struct drm_device *dev, struct drm_gem_object *gem_obj, int flags) { struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - struct dma_buf *dma_buf; exp_info.ops = &i915_dmabuf_ops; exp_info.size = gem_obj->size; exp_info.flags = flags; exp_info.priv = gem_obj; + exp_info.resv = obj->resv; if (obj->ops->dmabuf_export) { int ret = obj->ops->dmabuf_export(obj); @@ -283,30 +229,21 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev, return ERR_PTR(ret); } - dma_buf = drm_gem_dmabuf_export(dev, &exp_info); - if (IS_ERR(dma_buf)) - return dma_buf; - - export_fences(obj, dma_buf); - return dma_buf; + return drm_gem_dmabuf_export(dev, &exp_info); } -static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) { - struct sg_table *sg; - - sg = dma_buf_map_attachment(obj->base.import_attach, DMA_BIDIRECTIONAL); - if (IS_ERR(sg)) - return PTR_ERR(sg); - - obj->pages = sg; - return 0; + return dma_buf_map_attachment(obj->base.import_attach, + DMA_BIDIRECTIONAL); } -static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - dma_buf_unmap_attachment(obj->base.import_attach, - obj->pages, DMA_BIDIRECTIONAL); + dma_buf_unmap_attachment(obj->base.import_attach, pages, + DMA_BIDIRECTIONAL); } static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = { @@ -350,6 +287,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, drm_gem_private_object_init(dev, &obj->base, dma_buf->size); i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops); obj->base.import_attach = attach; + obj->resv = dma_buf->resv; /* We use GTT as shorthand for a coherent domain, one that is * neither in the GPU cache nor in the CPU cache, where all diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index b5e9e669f50f..bd08814b015c 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -33,14 +33,17 @@ #include "intel_drv.h" #include "i915_trace.h" -static bool -gpu_is_idle(struct drm_i915_private *dev_priv) +static bool ggtt_is_idle(struct drm_i915_private *dev_priv) { + struct i915_ggtt *ggtt = &dev_priv->ggtt; struct intel_engine_cs *engine; enum intel_engine_id id; for_each_engine(engine, dev_priv, id) { - if (intel_engine_is_active(engine)) + struct intel_timeline *tl; + + tl = &ggtt->base.timeline.engine[engine->id]; + if (i915_gem_active_isset(&tl->last_request)) return false; } @@ -56,7 +59,7 @@ mark_free(struct i915_vma *vma, unsigned int flags, struct list_head *unwind) if (WARN_ON(!list_empty(&vma->exec_list))) return false; - if (flags & PIN_NONFAULT && vma->obj->fault_mappable) + if (flags & PIN_NONFAULT && !list_empty(&vma->obj->userfault_link)) return false; list_add(&vma->exec_list, unwind); @@ -103,6 +106,7 @@ i915_gem_evict_something(struct i915_address_space *vm, struct i915_vma *vma, *next; int ret; + lockdep_assert_held(&vm->dev->struct_mutex); trace_i915_gem_evict(vm, min_size, alignment, flags); /* @@ -153,7 +157,7 @@ search_again: if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK) return -ENOSPC; - if (gpu_is_idle(dev_priv)) { + if (ggtt_is_idle(dev_priv)) { /* If we still have pending pageflip completions, drop * back to userspace to give our workqueues time to * acquire our locks and unpin the old scanouts. @@ -213,6 +217,8 @@ i915_gem_evict_for_vma(struct i915_vma *target) { struct drm_mm_node *node, *next; + lockdep_assert_held(&target->vm->dev->struct_mutex); + list_for_each_entry_safe(node, next, &target->vm->mm.head_node.node_list, node_list) { @@ -266,7 +272,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle) struct i915_vma *vma, *next; int ret; - WARN_ON(!mutex_is_locked(&vm->dev->struct_mutex)); + lockdep_assert_held(&vm->dev->struct_mutex); trace_i915_gem_evict_vm(vm); if (do_idle) { diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index e52affdcc125..097d9d8c2315 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -34,7 +34,6 @@ #include <drm/i915_drm.h> #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "i915_trace.h" #include "intel_drv.h" #include "intel_frontbuffer.h" @@ -288,7 +287,7 @@ static inline int use_cpu_reloc(struct drm_i915_gem_object *obj) if (DBG_USE_CPU_RELOC) return DBG_USE_CPU_RELOC > 0; - return (HAS_LLC(obj->base.dev) || + return (HAS_LLC(to_i915(obj->base.dev)) || obj->base.write_domain == I915_GEM_DOMAIN_CPU || obj->cache_level != I915_CACHE_NONE); } @@ -332,7 +331,8 @@ static void reloc_cache_init(struct reloc_cache *cache, cache->page = -1; cache->vaddr = 0; cache->i915 = i915; - cache->use_64bit_reloc = INTEL_GEN(cache->i915) >= 8; + /* Must be a variable in the struct to allow GCC to unroll. */ + cache->use_64bit_reloc = HAS_64BIT_RELOC(i915); cache->node.allocated = false; } @@ -418,15 +418,6 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj, unsigned long offset; void *vaddr; - if (cache->node.allocated) { - wmb(); - ggtt->base.insert_page(&ggtt->base, - i915_gem_object_get_dma_address(obj, page), - cache->node.start, I915_CACHE_NONE, 0); - cache->page = page; - return unmask_page(cache->vaddr); - } - if (cache->vaddr) { io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); } else { @@ -466,6 +457,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj, offset = cache->node.start; if (cache->node.allocated) { + wmb(); ggtt->base.insert_page(&ggtt->base, i915_gem_object_get_dma_address(obj, page), offset, I915_CACHE_NONE, 0); @@ -841,7 +833,7 @@ need_reloc_mappable(struct i915_vma *vma) return false; /* See also use_cpu_reloc() */ - if (HAS_LLC(vma->obj->base.dev)) + if (HAS_LLC(to_i915(vma->obj->base.dev))) return false; if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU) @@ -1109,44 +1101,20 @@ err: return ret; } -static unsigned int eb_other_engines(struct drm_i915_gem_request *req) -{ - unsigned int mask; - - mask = ~intel_engine_flag(req->engine) & I915_BO_ACTIVE_MASK; - mask <<= I915_BO_ACTIVE_SHIFT; - - return mask; -} - static int i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req, struct list_head *vmas) { - const unsigned int other_rings = eb_other_engines(req); struct i915_vma *vma; int ret; list_for_each_entry(vma, vmas, exec_list) { struct drm_i915_gem_object *obj = vma->obj; - struct reservation_object *resv; - - if (obj->flags & other_rings) { - ret = i915_gem_request_await_object - (req, obj, obj->base.pending_write_domain); - if (ret) - return ret; - } - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - ret = i915_sw_fence_await_reservation - (&req->submit, resv, &i915_fence_ops, - obj->base.pending_write_domain, 10*HZ, - GFP_KERNEL | __GFP_NOWARN); - if (ret < 0) - return ret; - } + ret = i915_gem_request_await_object + (req, obj, obj->base.pending_write_domain); + if (ret) + return ret; if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) i915_gem_clflush_object(obj, false); @@ -1279,6 +1247,12 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, return ctx; } +static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) +{ + return !(obj->cache_level == I915_CACHE_NONE || + obj->cache_level == I915_CACHE_WT); +} + void i915_vma_move_to_active(struct i915_vma *vma, struct drm_i915_gem_request *req, unsigned int flags) @@ -1288,8 +1262,6 @@ void i915_vma_move_to_active(struct i915_vma *vma, GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - obj->dirty = 1; /* be paranoid */ - /* Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is * vital here: mark_active adds to the start of the callback list, @@ -1297,37 +1269,31 @@ void i915_vma_move_to_active(struct i915_vma *vma, * add the active reference first and queue for it to be dropped * *last*. */ - if (!i915_gem_object_is_active(obj)) - i915_gem_object_get(obj); - i915_gem_object_set_active(obj, idx); - i915_gem_active_set(&obj->last_read[idx], req); + if (!i915_vma_is_active(vma)) + obj->active_count++; + i915_vma_set_active(vma, idx); + i915_gem_active_set(&vma->last_read[idx], req); + list_move_tail(&vma->vm_link, &vma->vm->active_list); if (flags & EXEC_OBJECT_WRITE) { - i915_gem_active_set(&obj->last_write, req); - - intel_fb_obj_invalidate(obj, ORIGIN_CS); + if (intel_fb_obj_invalidate(obj, ORIGIN_CS)) + i915_gem_active_set(&obj->frontbuffer_write, req); /* update for the implicit flush after a batch */ obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; + if (!obj->cache_dirty && gpu_write_needs_clflush(obj)) + obj->cache_dirty = true; } if (flags & EXEC_OBJECT_NEEDS_FENCE) i915_gem_active_set(&vma->last_fence, req); - - i915_vma_set_active(vma, idx); - i915_gem_active_set(&vma->last_read[idx], req); - list_move_tail(&vma->vm_link, &vma->vm->active_list); } static void eb_export_fence(struct drm_i915_gem_object *obj, struct drm_i915_gem_request *req, unsigned int flags) { - struct reservation_object *resv; - - resv = i915_gem_object_get_dmabuf_resv(obj); - if (!resv) - return; + struct reservation_object *resv = obj->resv; /* Ignore errors from failing to allocate the new fence, we can't * handle an error right now. Worst case should be missed @@ -1657,7 +1623,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, } if (args->flags & I915_EXEC_RESOURCE_STREAMER) { - if (!HAS_RESOURCE_STREAMER(dev)) { + if (!HAS_RESOURCE_STREAMER(dev_priv)) { DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n"); return -EINVAL; } @@ -1911,7 +1877,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; exec2_list[i].alignment = exec_list[i].alignment; exec2_list[i].offset = exec_list[i].offset; - if (INTEL_INFO(dev)->gen < 4) + if (INTEL_GEN(to_i915(dev)) < 4) exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; else exec2_list[i].flags = 0; diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence_reg.c index a6daf2deab74..0efa3571afc3 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence.c +++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.c @@ -343,6 +343,9 @@ i915_vma_get_fence(struct i915_vma *vma) struct drm_i915_fence_reg *fence; struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL; + /* Note that we revoke fences on runtime suspend. Therefore the user + * must keep the device awake whilst using the fence. + */ assert_rpm_wakelock_held(to_i915(vma->vm->dev)); /* Just update our place in the LRU if our fence is getting reused. */ @@ -365,22 +368,16 @@ i915_vma_get_fence(struct i915_vma *vma) /** * i915_gem_restore_fences - restore fence state - * @dev: DRM device + * @dev_priv: i915 device private * * Restore the hw fence state to match the software tracking again, to be called - * after a gpu reset and on resume. + * after a gpu reset and on resume. Note that on runtime suspend we only cancel + * the fences, to be reacquired by the user later. */ -void i915_gem_restore_fences(struct drm_device *dev) +void i915_gem_restore_fences(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); int i; - /* Note that this may be called outside of struct_mutex, by - * runtime suspend/resume. The barrier we require is enforced by - * rpm itself - all access to fences/GTT are only within an rpm - * wakeref, and to acquire that wakeref you must pass through here. - */ - for (i = 0; i < dev_priv->num_fence_regs; i++) { struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; struct i915_vma *vma = reg->vma; @@ -391,7 +388,7 @@ void i915_gem_restore_fences(struct drm_device *dev) */ if (vma && !i915_gem_object_is_tiled(vma->obj)) { GEM_BUG_ON(!reg->dirty); - GEM_BUG_ON(vma->obj->fault_mappable); + GEM_BUG_ON(!list_empty(&vma->obj->userfault_link)); list_move(®->link, &dev_priv->mm.fence_list); vma->fence = NULL; @@ -453,15 +450,14 @@ void i915_gem_restore_fences(struct drm_device *dev) /** * i915_gem_detect_bit_6_swizzle - detect bit 6 swizzling pattern - * @dev: DRM device + * @dev_priv: i915 device private * * Detects bit 6 swizzling of address lookup between IGD access and CPU * access through main memory. */ void -i915_gem_detect_bit_6_swizzle(struct drm_device *dev) +i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; @@ -475,7 +471,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev) */ swizzle_x = I915_BIT_6_SWIZZLE_NONE; swizzle_y = I915_BIT_6_SWIZZLE_NONE; - } else if (INTEL_INFO(dev)->gen >= 6) { + } else if (INTEL_GEN(dev_priv) >= 6) { if (dev_priv->preserve_bios_swizzle) { if (I915_READ(DISP_ARB_CTL) & DISP_TILE_SURFACE_SWIZZLING) { @@ -646,6 +642,7 @@ i915_gem_swizzle_page(struct page *page) /** * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling * @obj: i915 GEM buffer object + * @pages: the scattergather list of physical pages * * This function fixes up the swizzling in case any page frame number for this * object has changed in bit 17 since that state has been saved with @@ -656,7 +653,8 @@ i915_gem_swizzle_page(struct page *page) * by swapping them out and back in again). */ void -i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) +i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct sgt_iter sgt_iter; struct page *page; @@ -666,10 +664,9 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) return; i = 0; - for_each_sgt_page(page, sgt_iter, obj->pages) { + for_each_sgt_page(page, sgt_iter, pages) { char new_bit_17 = page_to_phys(page) >> 17; - if ((new_bit_17 & 0x1) != - (test_bit(i, obj->bit_17) != 0)) { + if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) { i915_gem_swizzle_page(page); set_page_dirty(page); } @@ -680,17 +677,19 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) /** * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling * @obj: i915 GEM buffer object + * @pages: the scattergather list of physical pages * * This function saves the bit 17 of each page frame number so that swizzling * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must * be called before the backing storage can be unpinned. */ void -i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) +i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages) { + const unsigned int page_count = obj->base.size >> PAGE_SHIFT; struct sgt_iter sgt_iter; struct page *page; - int page_count = obj->base.size >> PAGE_SHIFT; int i; if (obj->bit_17 == NULL) { @@ -705,7 +704,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i = 0; - for_each_sgt_page(page, sgt_iter, obj->pages) { + for_each_sgt_page(page, sgt_iter, pages) { if (page_to_phys(page) & (1 << 17)) __set_bit(i, obj->bit_17); else diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.h b/drivers/gpu/drm/i915/i915_gem_fence_reg.h index 91315557e421..22c4a2d01adf 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.h +++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.h @@ -1,5 +1,5 @@ /* - * Copyright 2016 Intel Corporation + * Copyright © 2016 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -17,29 +17,35 @@ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. * */ -#ifndef _I915_GEM_DMABUF_H_ -#define _I915_GEM_DMABUF_H_ +#ifndef __I915_FENCE_REG_H__ +#define __I915_FENCE_REG_H__ -#include <linux/dma-buf.h> +#include <linux/list.h> -static inline struct reservation_object * -i915_gem_object_get_dmabuf_resv(struct drm_i915_gem_object *obj) -{ - struct dma_buf *dma_buf; +struct drm_i915_private; +struct i915_vma; - if (obj->base.dma_buf) - dma_buf = obj->base.dma_buf; - else if (obj->base.import_attach) - dma_buf = obj->base.import_attach->dmabuf; - else - return NULL; - - return dma_buf->resv; -} +struct drm_i915_fence_reg { + struct list_head link; + struct drm_i915_private *i915; + struct i915_vma *vma; + int pin_count; + int id; + /** + * Whether the tiling parameters for the currently + * associated fence register have changed. Note that + * for the purposes of tracking tiling changes we also + * treat the unfenced register, the register slot that + * the object occupies whilst it executes a fenced + * command (such as BLT on gen2/3), as a "fence". + */ + bool dirty; +}; #endif + diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 062fb0ad75da..b4bde1452f2a 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -31,6 +31,7 @@ #include "i915_vgpu.h" #include "i915_trace.h" #include "intel_drv.h" +#include "intel_frontbuffer.h" #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM) @@ -95,13 +96,6 @@ * */ -static inline struct i915_ggtt * -i915_vm_to_ggtt(struct i915_address_space *vm) -{ - GEM_BUG_ON(!i915_is_ggtt(vm)); - return container_of(vm, struct i915_ggtt, base); -} - static int i915_get_ggtt_vma_pages(struct i915_vma *vma); @@ -175,7 +169,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma, { u32 pte_flags = 0; - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages; /* Currently applicable only to VLV */ if (vma->obj->gt_ro) @@ -326,10 +320,10 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr, return pte; } -static int __setup_page_dma(struct drm_device *dev, +static int __setup_page_dma(struct drm_i915_private *dev_priv, struct i915_page_dma *p, gfp_t flags) { - struct device *kdev = &dev->pdev->dev; + struct device *kdev = &dev_priv->drm.pdev->dev; p->page = alloc_page(flags); if (!p->page) @@ -346,14 +340,16 @@ static int __setup_page_dma(struct drm_device *dev, return 0; } -static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) +static int setup_page_dma(struct drm_i915_private *dev_priv, + struct i915_page_dma *p) { - return __setup_page_dma(dev, p, I915_GFP_DMA); + return __setup_page_dma(dev_priv, p, I915_GFP_DMA); } -static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) +static void cleanup_page_dma(struct drm_i915_private *dev_priv, + struct i915_page_dma *p) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; if (WARN_ON(!p->page)) return; @@ -386,8 +382,8 @@ static void kunmap_page_dma(struct drm_i915_private *dev_priv, void *vaddr) #define kunmap_px(ppgtt, vaddr) \ kunmap_page_dma(to_i915((ppgtt)->base.dev), (vaddr)) -#define setup_px(dev, px) setup_page_dma((dev), px_base(px)) -#define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) +#define setup_px(dev_priv, px) setup_page_dma((dev_priv), px_base(px)) +#define cleanup_px(dev_priv, px) cleanup_page_dma((dev_priv), px_base(px)) #define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v)) #define fill32_px(dev_priv, px, v) \ fill_page_dma_32((dev_priv), px_base(px), (v)) @@ -415,24 +411,23 @@ static void fill_page_dma_32(struct drm_i915_private *dev_priv, } static int -setup_scratch_page(struct drm_device *dev, +setup_scratch_page(struct drm_i915_private *dev_priv, struct i915_page_dma *scratch, gfp_t gfp) { - return __setup_page_dma(dev, scratch, gfp | __GFP_ZERO); + return __setup_page_dma(dev_priv, scratch, gfp | __GFP_ZERO); } -static void cleanup_scratch_page(struct drm_device *dev, +static void cleanup_scratch_page(struct drm_i915_private *dev_priv, struct i915_page_dma *scratch) { - cleanup_page_dma(dev, scratch); + cleanup_page_dma(dev_priv, scratch); } -static struct i915_page_table *alloc_pt(struct drm_device *dev) +static struct i915_page_table *alloc_pt(struct drm_i915_private *dev_priv) { struct i915_page_table *pt; - const size_t count = INTEL_INFO(dev)->gen >= 8 ? - GEN8_PTES : GEN6_PTES; + const size_t count = INTEL_GEN(dev_priv) >= 8 ? GEN8_PTES : GEN6_PTES; int ret = -ENOMEM; pt = kzalloc(sizeof(*pt), GFP_KERNEL); @@ -445,7 +440,7 @@ static struct i915_page_table *alloc_pt(struct drm_device *dev) if (!pt->used_ptes) goto fail_bitmap; - ret = setup_px(dev, pt); + ret = setup_px(dev_priv, pt); if (ret) goto fail_page_m; @@ -459,9 +454,10 @@ fail_bitmap: return ERR_PTR(ret); } -static void free_pt(struct drm_device *dev, struct i915_page_table *pt) +static void free_pt(struct drm_i915_private *dev_priv, + struct i915_page_table *pt) { - cleanup_px(dev, pt); + cleanup_px(dev_priv, pt); kfree(pt->used_ptes); kfree(pt); } @@ -490,7 +486,7 @@ static void gen6_initialize_pt(struct i915_address_space *vm, fill32_px(to_i915(vm->dev), pt, scratch_pte); } -static struct i915_page_directory *alloc_pd(struct drm_device *dev) +static struct i915_page_directory *alloc_pd(struct drm_i915_private *dev_priv) { struct i915_page_directory *pd; int ret = -ENOMEM; @@ -504,7 +500,7 @@ static struct i915_page_directory *alloc_pd(struct drm_device *dev) if (!pd->used_pdes) goto fail_bitmap; - ret = setup_px(dev, pd); + ret = setup_px(dev_priv, pd); if (ret) goto fail_page_m; @@ -518,10 +514,11 @@ fail_bitmap: return ERR_PTR(ret); } -static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) +static void free_pd(struct drm_i915_private *dev_priv, + struct i915_page_directory *pd) { if (px_page(pd)) { - cleanup_px(dev, pd); + cleanup_px(dev_priv, pd); kfree(pd->used_pdes); kfree(pd); } @@ -537,10 +534,10 @@ static void gen8_initialize_pd(struct i915_address_space *vm, fill_px(to_i915(vm->dev), pd, scratch_pde); } -static int __pdp_init(struct drm_device *dev, +static int __pdp_init(struct drm_i915_private *dev_priv, struct i915_page_directory_pointer *pdp) { - size_t pdpes = I915_PDPES_PER_PDP(dev); + size_t pdpes = I915_PDPES_PER_PDP(dev_priv); pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), @@ -569,22 +566,22 @@ static void __pdp_fini(struct i915_page_directory_pointer *pdp) } static struct -i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) +i915_page_directory_pointer *alloc_pdp(struct drm_i915_private *dev_priv) { struct i915_page_directory_pointer *pdp; int ret = -ENOMEM; - WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); + WARN_ON(!USES_FULL_48BIT_PPGTT(dev_priv)); pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); if (!pdp) return ERR_PTR(-ENOMEM); - ret = __pdp_init(dev, pdp); + ret = __pdp_init(dev_priv, pdp); if (ret) goto fail_bitmap; - ret = setup_px(dev, pdp); + ret = setup_px(dev_priv, pdp); if (ret) goto fail_page_m; @@ -598,12 +595,12 @@ fail_bitmap: return ERR_PTR(ret); } -static void free_pdp(struct drm_device *dev, +static void free_pdp(struct drm_i915_private *dev_priv, struct i915_page_directory_pointer *pdp) { __pdp_fini(pdp); - if (USES_FULL_48BIT_PPGTT(dev)) { - cleanup_px(dev, pdp); + if (USES_FULL_48BIT_PPGTT(dev_priv)) { + cleanup_px(dev_priv, pdp); kfree(pdp); } } @@ -637,7 +634,7 @@ gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt, { gen8_ppgtt_pdpe_t *page_directorypo; - if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) + if (!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev))) return; page_directorypo = kmap_px(pdp); @@ -653,7 +650,7 @@ gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt, { gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); - WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); + WARN_ON(!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev))); pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); kunmap_px(ppgtt, pagemap); } @@ -706,6 +703,16 @@ static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); } +/* PDE TLBs are a pain to invalidate on GEN8+. When we modify + * the page table structures, we mark them dirty so that + * context switching/execlist queuing code takes extra steps + * to ensure that tlbs are flushed. + */ +static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) +{ + ppgtt->pd_dirty_rings = INTEL_INFO(to_i915(ppgtt->base.dev))->ring_mask; +} + /* Removes entries from a single page table, releasing it if it's empty. * Caller can use the return value to update higher-level entries. */ @@ -715,9 +722,9 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, uint64_t length) { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - unsigned int pte_start = gen8_pte_index(start); unsigned int num_entries = gen8_pte_count(start, length); - uint64_t pte; + unsigned int pte = gen8_pte_index(start); + unsigned int pte_end = pte + num_entries; gen8_pte_t *pt_vaddr; gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); @@ -725,17 +732,19 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, if (WARN_ON(!px_page(pt))) return false; - bitmap_clear(pt->used_ptes, pte_start, num_entries); + GEM_BUG_ON(pte_end > GEN8_PTES); + + bitmap_clear(pt->used_ptes, pte, num_entries); if (bitmap_empty(pt->used_ptes, GEN8_PTES)) { - free_pt(vm->dev, pt); + free_pt(to_i915(vm->dev), pt); return true; } pt_vaddr = kmap_px(pt); - for (pte = pte_start; pte < num_entries; pte++) - pt_vaddr[pte] = scratch_pte; + while (pte < pte_end) + pt_vaddr[pte++] = scratch_pte; kunmap_px(ppgtt, pt_vaddr); @@ -770,7 +779,7 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, } if (bitmap_empty(pd->used_pdes, I915_PDES)) { - free_pd(vm->dev, pd); + free_pd(to_i915(vm->dev), pd); return true; } @@ -786,6 +795,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, uint64_t length) { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_page_directory *pd; uint64_t pdpe; gen8_ppgtt_pdpe_t *pdpe_vaddr; @@ -798,7 +808,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, if (gen8_ppgtt_clear_pd(vm, pd, start, length)) { __clear_bit(pdpe, pdp->used_pdpes); - if (USES_FULL_48BIT_PPGTT(vm->dev)) { + if (USES_FULL_48BIT_PPGTT(dev_priv)) { pdpe_vaddr = kmap_px(pdp); pdpe_vaddr[pdpe] = scratch_pdpe; kunmap_px(ppgtt, pdpe_vaddr); @@ -806,9 +816,11 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, } } - if (USES_FULL_48BIT_PPGTT(vm->dev) && - bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(vm->dev))) { - free_pdp(vm->dev, pdp); + mark_tlbs_dirty(ppgtt); + + if (USES_FULL_48BIT_PPGTT(dev_priv) && + bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv))) { + free_pdp(dev_priv, pdp); return true; } @@ -831,7 +843,7 @@ static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm, gen8_ppgtt_pml4e_t scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC); - GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(vm->dev)); + GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(to_i915(vm->dev))); gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { if (WARN_ON(!pml4->pdps[pml4e])) @@ -851,7 +863,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - if (USES_FULL_48BIT_PPGTT(vm->dev)) + if (USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length); else gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length); @@ -886,7 +898,7 @@ gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, kunmap_px(ppgtt, pt_vaddr); pt_vaddr = NULL; if (++pde == I915_PDES) { - if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) + if (++pdpe == I915_PDPES_PER_PDP(to_i915(vm->dev))) break; pde = 0; } @@ -909,7 +921,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); - if (!USES_FULL_48BIT_PPGTT(vm->dev)) { + if (!USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) { gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, cache_level); } else { @@ -924,7 +936,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables(struct drm_device *dev, +static void gen8_free_page_tables(struct drm_i915_private *dev_priv, struct i915_page_directory *pd) { int i; @@ -936,34 +948,34 @@ static void gen8_free_page_tables(struct drm_device *dev, if (WARN_ON(!pd->page_table[i])) continue; - free_pt(dev, pd->page_table[i]); + free_pt(dev_priv, pd->page_table[i]); pd->page_table[i] = NULL; } } static int gen8_init_scratch(struct i915_address_space *vm) { - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); int ret; - ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA); + ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA); if (ret) return ret; - vm->scratch_pt = alloc_pt(dev); + vm->scratch_pt = alloc_pt(dev_priv); if (IS_ERR(vm->scratch_pt)) { ret = PTR_ERR(vm->scratch_pt); goto free_scratch_page; } - vm->scratch_pd = alloc_pd(dev); + vm->scratch_pd = alloc_pd(dev_priv); if (IS_ERR(vm->scratch_pd)) { ret = PTR_ERR(vm->scratch_pd); goto free_pt; } - if (USES_FULL_48BIT_PPGTT(dev)) { - vm->scratch_pdp = alloc_pdp(dev); + if (USES_FULL_48BIT_PPGTT(dev_priv)) { + vm->scratch_pdp = alloc_pdp(dev_priv); if (IS_ERR(vm->scratch_pdp)) { ret = PTR_ERR(vm->scratch_pdp); goto free_pd; @@ -972,17 +984,17 @@ static int gen8_init_scratch(struct i915_address_space *vm) gen8_initialize_pt(vm, vm->scratch_pt); gen8_initialize_pd(vm, vm->scratch_pd); - if (USES_FULL_48BIT_PPGTT(dev)) + if (USES_FULL_48BIT_PPGTT(dev_priv)) gen8_initialize_pdp(vm, vm->scratch_pdp); return 0; free_pd: - free_pd(dev, vm->scratch_pd); + free_pd(dev_priv, vm->scratch_pd); free_pt: - free_pt(dev, vm->scratch_pt); + free_pt(dev_priv, vm->scratch_pt); free_scratch_page: - cleanup_scratch_page(dev, &vm->scratch_page); + cleanup_scratch_page(dev_priv, &vm->scratch_page); return ret; } @@ -1020,54 +1032,56 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) static void gen8_free_scratch(struct i915_address_space *vm) { - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); - if (USES_FULL_48BIT_PPGTT(dev)) - free_pdp(dev, vm->scratch_pdp); - free_pd(dev, vm->scratch_pd); - free_pt(dev, vm->scratch_pt); - cleanup_scratch_page(dev, &vm->scratch_page); + if (USES_FULL_48BIT_PPGTT(dev_priv)) + free_pdp(dev_priv, vm->scratch_pdp); + free_pd(dev_priv, vm->scratch_pd); + free_pt(dev_priv, vm->scratch_pt); + cleanup_scratch_page(dev_priv, &vm->scratch_page); } -static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, +static void gen8_ppgtt_cleanup_3lvl(struct drm_i915_private *dev_priv, struct i915_page_directory_pointer *pdp) { int i; - for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { + for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)) { if (WARN_ON(!pdp->page_directory[i])) continue; - gen8_free_page_tables(dev, pdp->page_directory[i]); - free_pd(dev, pdp->page_directory[i]); + gen8_free_page_tables(dev_priv, pdp->page_directory[i]); + free_pd(dev_priv, pdp->page_directory[i]); } - free_pdp(dev, pdp); + free_pdp(dev_priv, pdp); } static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) { + struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); int i; for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { if (WARN_ON(!ppgtt->pml4.pdps[i])) continue; - gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); + gen8_ppgtt_cleanup_3lvl(dev_priv, ppgtt->pml4.pdps[i]); } - cleanup_px(ppgtt->base.dev, &ppgtt->pml4); + cleanup_px(dev_priv, &ppgtt->pml4); } static void gen8_ppgtt_cleanup(struct i915_address_space *vm) { + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - if (intel_vgpu_active(to_i915(vm->dev))) + if (intel_vgpu_active(dev_priv)) gen8_ppgtt_notify_vgt(ppgtt, false); - if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) - gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); + if (!USES_FULL_48BIT_PPGTT(dev_priv)) + gen8_ppgtt_cleanup_3lvl(dev_priv, &ppgtt->pdp); else gen8_ppgtt_cleanup_4lvl(ppgtt); @@ -1098,7 +1112,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, uint64_t length, unsigned long *new_pts) { - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_page_table *pt; uint32_t pde; @@ -1110,7 +1124,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, continue; } - pt = alloc_pt(dev); + pt = alloc_pt(dev_priv); if (IS_ERR(pt)) goto unwind_out; @@ -1124,7 +1138,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, unwind_out: for_each_set_bit(pde, new_pts, I915_PDES) - free_pt(dev, pd->page_table[pde]); + free_pt(dev_priv, pd->page_table[pde]); return -ENOMEM; } @@ -1159,10 +1173,10 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, uint64_t length, unsigned long *new_pds) { - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_page_directory *pd; uint32_t pdpe; - uint32_t pdpes = I915_PDPES_PER_PDP(dev); + uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv); WARN_ON(!bitmap_empty(new_pds, pdpes)); @@ -1170,7 +1184,7 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, if (test_bit(pdpe, pdp->used_pdpes)) continue; - pd = alloc_pd(dev); + pd = alloc_pd(dev_priv); if (IS_ERR(pd)) goto unwind_out; @@ -1184,7 +1198,7 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, unwind_out: for_each_set_bit(pdpe, new_pds, pdpes) - free_pd(dev, pdp->page_directory[pdpe]); + free_pd(dev_priv, pdp->page_directory[pdpe]); return -ENOMEM; } @@ -1212,7 +1226,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, uint64_t length, unsigned long *new_pdps) { - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_page_directory_pointer *pdp; uint32_t pml4e; @@ -1220,7 +1234,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { if (!test_bit(pml4e, pml4->used_pml4es)) { - pdp = alloc_pdp(dev); + pdp = alloc_pdp(dev_priv); if (IS_ERR(pdp)) goto unwind_out; @@ -1238,7 +1252,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, unwind_out: for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) - free_pdp(dev, pml4->pdps[pml4e]); + free_pdp(dev_priv, pml4->pdps[pml4e]); return -ENOMEM; } @@ -1280,16 +1294,6 @@ err_out: return -ENOMEM; } -/* PDE TLBs are a pain to invalidate on GEN8+. When we modify - * the page table structures, we mark them dirty so that - * context switching/execlist queuing code takes extra steps - * to ensure that tlbs are flushed. - */ -static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) -{ - ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; -} - static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, struct i915_page_directory_pointer *pdp, uint64_t start, @@ -1297,12 +1301,12 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); unsigned long *new_page_dirs, *new_page_tables; - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_page_directory *pd; const uint64_t orig_start = start; const uint64_t orig_length = length; uint32_t pdpe; - uint32_t pdpes = I915_PDPES_PER_PDP(dev); + uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv); int ret; /* Wrap is never okay since we can only represent 48b, and we don't @@ -1390,11 +1394,12 @@ err_out: for_each_set_bit(temp, new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES), I915_PDES) - free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); + free_pt(dev_priv, + pdp->page_directory[pdpe]->page_table[temp]); } for_each_set_bit(pdpe, new_page_dirs, pdpes) - free_pd(dev, pdp->page_directory[pdpe]); + free_pd(dev_priv, pdp->page_directory[pdpe]); free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); mark_tlbs_dirty(ppgtt); @@ -1445,7 +1450,7 @@ static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, err_out: for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) - gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); + gen8_ppgtt_cleanup_3lvl(to_i915(vm->dev), pml4->pdps[pml4e]); return ret; } @@ -1455,7 +1460,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - if (USES_FULL_48BIT_PPGTT(vm->dev)) + if (USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); else return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); @@ -1526,7 +1531,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); - if (!USES_FULL_48BIT_PPGTT(vm->dev)) { + if (!USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) { gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); } else { uint64_t pml4e; @@ -1546,7 +1551,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) { unsigned long *new_page_dirs, *new_page_tables; - uint32_t pdpes = I915_PDPES_PER_PDP(dev); + uint32_t pdpes = I915_PDPES_PER_PDP(to_i915(ppgtt->base.dev)); int ret; /* We allocate temp bitmap for page tables for no gain @@ -1579,6 +1584,7 @@ static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) */ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) { + struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); int ret; ret = gen8_init_scratch(&ppgtt->base); @@ -1594,8 +1600,8 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.bind_vma = ppgtt_bind_vma; ppgtt->debug_dump = gen8_dump_ppgtt; - if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { - ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); + if (USES_FULL_48BIT_PPGTT(dev_priv)) { + ret = setup_px(dev_priv, &ppgtt->pml4); if (ret) goto free_scratch; @@ -1604,7 +1610,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.total = 1ULL << 48; ppgtt->switch_mm = gen8_48b_mm_switch; } else { - ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); + ret = __pdp_init(dev_priv, &ppgtt->pdp); if (ret) goto free_scratch; @@ -1614,14 +1620,14 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 0, 0, GEN8_PML4E_SHIFT); - if (intel_vgpu_active(to_i915(ppgtt->base.dev))) { + if (intel_vgpu_active(dev_priv)) { ret = gen8_preallocate_top_level_pdps(ppgtt); if (ret) goto free_scratch; } } - if (intel_vgpu_active(to_i915(ppgtt->base.dev))) + if (intel_vgpu_active(dev_priv)) gen8_ppgtt_notify_vgt(ppgtt, true); return 0; @@ -1796,22 +1802,21 @@ static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, return 0; } -static void gen8_ppgtt_enable(struct drm_device *dev) +static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; enum intel_engine_id id; for_each_engine(engine, dev_priv, id) { - u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; + u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ? + GEN8_GFX_PPGTT_48B : 0; I915_WRITE(RING_MODE_GEN7(engine), _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); } } -static void gen7_ppgtt_enable(struct drm_device *dev) +static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; uint32_t ecochk, ecobits; enum intel_engine_id id; @@ -1835,9 +1840,8 @@ static void gen7_ppgtt_enable(struct drm_device *dev) } } -static void gen6_ppgtt_enable(struct drm_device *dev) +static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t ecochk, gab_ctl, ecobits; ecobits = I915_READ(GAC_ECO_BITS); @@ -1923,8 +1927,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, uint64_t start_in, uint64_t length_in) { DECLARE_BITMAP(new_page_tables, I915_PDES); - struct drm_device *dev = vm->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_page_table *pt; @@ -1954,7 +1957,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, /* We've already allocated a page table */ WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); - pt = alloc_pt(dev); + pt = alloc_pt(dev_priv); if (IS_ERR(pt)) { ret = PTR_ERR(pt); goto unwind_out; @@ -2002,7 +2005,7 @@ unwind_out: struct i915_page_table *pt = ppgtt->pd.page_table[pde]; ppgtt->pd.page_table[pde] = vm->scratch_pt; - free_pt(vm->dev, pt); + free_pt(dev_priv, pt); } mark_tlbs_dirty(ppgtt); @@ -2011,16 +2014,16 @@ unwind_out: static int gen6_init_scratch(struct i915_address_space *vm) { - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); int ret; - ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA); + ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA); if (ret) return ret; - vm->scratch_pt = alloc_pt(dev); + vm->scratch_pt = alloc_pt(dev_priv); if (IS_ERR(vm->scratch_pt)) { - cleanup_scratch_page(dev, &vm->scratch_page); + cleanup_scratch_page(dev_priv, &vm->scratch_page); return PTR_ERR(vm->scratch_pt); } @@ -2031,17 +2034,17 @@ static int gen6_init_scratch(struct i915_address_space *vm) static void gen6_free_scratch(struct i915_address_space *vm) { - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); - free_pt(dev, vm->scratch_pt); - cleanup_scratch_page(dev, &vm->scratch_page); + free_pt(dev_priv, vm->scratch_pt); + cleanup_scratch_page(dev_priv, &vm->scratch_page); } static void gen6_ppgtt_cleanup(struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_page_directory *pd = &ppgtt->pd; - struct drm_device *dev = vm->dev; + struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_page_table *pt; uint32_t pde; @@ -2049,7 +2052,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) gen6_for_all_pdes(pt, pd, pde) if (pt != vm->scratch_pt) - free_pt(dev, pt); + free_pt(dev_priv, pt); gen6_free_scratch(vm); } @@ -2057,8 +2060,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) { struct i915_address_space *vm = &ppgtt->base; - struct drm_device *dev = ppgtt->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; bool retried = false; int ret; @@ -2123,8 +2125,7 @@ static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) { - struct drm_device *dev = ppgtt->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; int ret; @@ -2184,8 +2185,10 @@ static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt, } static void i915_address_space_init(struct i915_address_space *vm, - struct drm_i915_private *dev_priv) + struct drm_i915_private *dev_priv, + const char *name) { + i915_gem_timeline_init(dev_priv, &vm->timeline, name); drm_mm_init(&vm->mm, vm->start, vm->total); INIT_LIST_HEAD(&vm->active_list); INIT_LIST_HEAD(&vm->inactive_list); @@ -2193,10 +2196,15 @@ static void i915_address_space_init(struct i915_address_space *vm, list_add_tail(&vm->global_link, &dev_priv->vm_list); } -static void gtt_write_workarounds(struct drm_device *dev) +static void i915_address_space_fini(struct i915_address_space *vm) { - struct drm_i915_private *dev_priv = to_i915(dev); + i915_gem_timeline_fini(&vm->timeline); + drm_mm_takedown(&vm->mm); + list_del(&vm->global_link); +} +static void gtt_write_workarounds(struct drm_i915_private *dev_priv) +{ /* This function is for gtt related workarounds. This function is * called on driver load and after a GPU reset, so you can place * workarounds here even if they get overwritten by GPU reset. @@ -2214,25 +2222,24 @@ static void gtt_write_workarounds(struct drm_device *dev) static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt, struct drm_i915_private *dev_priv, - struct drm_i915_file_private *file_priv) + struct drm_i915_file_private *file_priv, + const char *name) { int ret; ret = __hw_ppgtt_init(ppgtt, dev_priv); if (ret == 0) { kref_init(&ppgtt->ref); - i915_address_space_init(&ppgtt->base, dev_priv); + i915_address_space_init(&ppgtt->base, dev_priv, name); ppgtt->base.file = file_priv; } return ret; } -int i915_ppgtt_init_hw(struct drm_device *dev) +int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - gtt_write_workarounds(dev); + gtt_write_workarounds(dev_priv); /* In the case of execlists, PPGTT is enabled by the context descriptor * and the PDPs are contained within the context itself. We don't @@ -2240,24 +2247,25 @@ int i915_ppgtt_init_hw(struct drm_device *dev) if (i915.enable_execlists) return 0; - if (!USES_PPGTT(dev)) + if (!USES_PPGTT(dev_priv)) return 0; if (IS_GEN6(dev_priv)) - gen6_ppgtt_enable(dev); + gen6_ppgtt_enable(dev_priv); else if (IS_GEN7(dev_priv)) - gen7_ppgtt_enable(dev); - else if (INTEL_INFO(dev)->gen >= 8) - gen8_ppgtt_enable(dev); + gen7_ppgtt_enable(dev_priv); + else if (INTEL_GEN(dev_priv) >= 8) + gen8_ppgtt_enable(dev_priv); else - MISSING_CASE(INTEL_INFO(dev)->gen); + MISSING_CASE(INTEL_GEN(dev_priv)); return 0; } struct i915_hw_ppgtt * i915_ppgtt_create(struct drm_i915_private *dev_priv, - struct drm_i915_file_private *fpriv) + struct drm_i915_file_private *fpriv, + const char *name) { struct i915_hw_ppgtt *ppgtt; int ret; @@ -2266,7 +2274,7 @@ i915_ppgtt_create(struct drm_i915_private *dev_priv, if (!ppgtt) return ERR_PTR(-ENOMEM); - ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv); + ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv, name); if (ret) { kfree(ppgtt); return ERR_PTR(ret); @@ -2277,7 +2285,7 @@ i915_ppgtt_create(struct drm_i915_private *dev_priv, return ppgtt; } -void i915_ppgtt_release(struct kref *kref) +void i915_ppgtt_release(struct kref *kref) { struct i915_hw_ppgtt *ppgtt = container_of(kref, struct i915_hw_ppgtt, ref); @@ -2289,8 +2297,7 @@ void i915_ppgtt_release(struct kref *kref) WARN_ON(!list_empty(&ppgtt->base.inactive_list)); WARN_ON(!list_empty(&ppgtt->base.unbound_list)); - list_del(&ppgtt->base.global_link); - drm_mm_takedown(&ppgtt->base.mm); + i915_address_space_fini(&ppgtt->base); ppgtt->base.cleanup(&ppgtt->base); kfree(ppgtt); @@ -2352,15 +2359,14 @@ static void i915_ggtt_flush(struct drm_i915_private *dev_priv) } } -void i915_gem_suspend_gtt_mappings(struct drm_device *dev) +void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; /* Don't bother messing with faults pre GEN6 as we have little * documentation supporting that it's a good idea. */ - if (INTEL_INFO(dev)->gen < 6) + if (INTEL_GEN(dev_priv) < 6) return; i915_check_and_clear_faults(dev_priv); @@ -2370,14 +2376,15 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev) i915_ggtt_flush(dev_priv); } -int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) +int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - if (!dma_map_sg(&obj->base.dev->pdev->dev, - obj->pages->sgl, obj->pages->nents, - PCI_DMA_BIDIRECTIONAL)) - return -ENOSPC; + if (dma_map_sg(&obj->base.dev->pdev->dev, + pages->sgl, pages->nents, + PCI_DMA_BIDIRECTIONAL)) + return 0; - return 0; + return -ENOSPC; } static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) @@ -2395,16 +2402,11 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm, gen8_pte_t __iomem *pte = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + (offset >> PAGE_SHIFT); - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); gen8_set_pte(pte, gen8_pte_encode(addr, level)); I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void gen8_ggtt_insert_entries(struct i915_address_space *vm, @@ -2418,11 +2420,8 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, gen8_pte_t __iomem *gtt_entries; gen8_pte_t gtt_entry; dma_addr_t addr; - int rpm_atomic_seq; int i = 0; - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT); for_each_sgt_dma(addr, sgt_iter, st) { @@ -2446,8 +2445,6 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, */ I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } struct insert_entries { @@ -2486,16 +2483,11 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm, gen6_pte_t __iomem *pte = (gen6_pte_t __iomem *)dev_priv->ggtt.gsm + (offset >> PAGE_SHIFT); - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); iowrite32(vm->pte_encode(addr, level, flags), pte); I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } /* @@ -2515,11 +2507,8 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, gen6_pte_t __iomem *gtt_entries; gen6_pte_t gtt_entry; dma_addr_t addr; - int rpm_atomic_seq; int i = 0; - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT); for_each_sgt_dma(addr, sgt_iter, st) { @@ -2542,8 +2531,6 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, */ I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void nop_clear_range(struct i915_address_space *vm, @@ -2554,7 +2541,6 @@ static void nop_clear_range(struct i915_address_space *vm, static void gen8_ggtt_clear_range(struct i915_address_space *vm, uint64_t start, uint64_t length) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; @@ -2562,9 +2548,6 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, (gen8_pte_t __iomem *)ggtt->gsm + first_entry; const int max_entries = ggtt_total_entries(ggtt) - first_entry; int i; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); if (WARN(num_entries > max_entries, "First entry = %d; Num entries = %d (max=%d)\n", @@ -2576,15 +2559,12 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, for (i = 0; i < num_entries; i++) gen8_set_pte(>t_base[i], scratch_pte); readl(gtt_base); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void gen6_ggtt_clear_range(struct i915_address_space *vm, uint64_t start, uint64_t length) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; @@ -2592,9 +2572,6 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, (gen6_pte_t __iomem *)ggtt->gsm + first_entry; const int max_entries = ggtt_total_entries(ggtt) - first_entry; int i; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); if (WARN(num_entries > max_entries, "First entry = %d; Num entries = %d (max=%d)\n", @@ -2607,8 +2584,6 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, for (i = 0; i < num_entries; i++) iowrite32(scratch_pte, >t_base[i]); readl(gtt_base); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void i915_ggtt_insert_page(struct i915_address_space *vm, @@ -2617,16 +2592,10 @@ static void i915_ggtt_insert_page(struct i915_address_space *vm, enum i915_cache_level cache_level, u32 unused) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); unsigned int flags = (cache_level == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void i915_ggtt_insert_entries(struct i915_address_space *vm, @@ -2634,39 +2603,25 @@ static void i915_ggtt_insert_entries(struct i915_address_space *vm, uint64_t start, enum i915_cache_level cache_level, u32 unused) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); unsigned int flags = (cache_level == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); - } static void i915_ggtt_clear_range(struct i915_address_space *vm, uint64_t start, uint64_t length) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); - unsigned first_entry = start >> PAGE_SHIFT; - unsigned num_entries = length >> PAGE_SHIFT; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - - intel_gtt_clear_range(first_entry, num_entries); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); + intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); } static int ggtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { + struct drm_i915_private *i915 = to_i915(vma->vm->dev); struct drm_i915_gem_object *obj = vma->obj; u32 pte_flags = 0; int ret; @@ -2679,8 +2634,10 @@ static int ggtt_bind_vma(struct i915_vma *vma, if (obj->gt_ro) pte_flags |= PTE_READ_ONLY; + intel_runtime_pm_get(i915); vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, cache_level, pte_flags); + intel_runtime_pm_put(i915); /* * Without aliasing PPGTT there's no difference between @@ -2696,6 +2653,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { + struct drm_i915_private *i915 = to_i915(vma->vm->dev); u32 pte_flags; int ret; @@ -2710,14 +2668,15 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, if (flags & I915_VMA_GLOBAL_BIND) { + intel_runtime_pm_get(i915); vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, cache_level, pte_flags); + intel_runtime_pm_put(i915); } if (flags & I915_VMA_LOCAL_BIND) { - struct i915_hw_ppgtt *appgtt = - to_i915(vma->vm->dev)->mm.aliasing_ppgtt; + struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; appgtt->base.insert_entries(&appgtt->base, vma->pages, vma->node.start, cache_level, pte_flags); @@ -2728,19 +2687,24 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, static void ggtt_unbind_vma(struct i915_vma *vma) { - struct i915_hw_ppgtt *appgtt = to_i915(vma->vm->dev)->mm.aliasing_ppgtt; + struct drm_i915_private *i915 = to_i915(vma->vm->dev); + struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; const u64 size = min(vma->size, vma->node.size); - if (vma->flags & I915_VMA_GLOBAL_BIND) + if (vma->flags & I915_VMA_GLOBAL_BIND) { + intel_runtime_pm_get(i915); vma->vm->clear_range(vma->vm, vma->node.start, size); + intel_runtime_pm_put(i915); + } if (vma->flags & I915_VMA_LOCAL_BIND && appgtt) appgtt->base.clear_range(&appgtt->base, vma->node.start, size); } -void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) +void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); struct device *kdev = &dev_priv->drm.pdev->dev; @@ -2754,8 +2718,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } - dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents, - PCI_DMA_BIDIRECTIONAL); + dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL); } static void i915_gtt_color_adjust(struct drm_mm_node *node, @@ -2875,8 +2838,9 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv) if (drm_mm_initialized(&ggtt->base.mm)) { intel_vgt_deballoon(dev_priv); - drm_mm_takedown(&ggtt->base.mm); - list_del(&ggtt->base.global_link); + mutex_lock(&dev_priv->drm.struct_mutex); + i915_address_space_fini(&ggtt->base); + mutex_unlock(&dev_priv->drm.struct_mutex); } ggtt->base.cleanup(&ggtt->base); @@ -2965,6 +2929,7 @@ static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) { + struct drm_i915_private *dev_priv = to_i915(ggtt->base.dev); struct pci_dev *pdev = ggtt->base.dev->pdev; phys_addr_t phys_addr; int ret; @@ -2979,7 +2944,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) * resort to an uncached mapping. The WC issue is easily caught by the * readback check when writing GTT PTE entries. */ - if (IS_BROXTON(to_i915(ggtt->base.dev))) + if (IS_BROXTON(dev_priv)) ggtt->gsm = ioremap_nocache(phys_addr, size); else ggtt->gsm = ioremap_wc(phys_addr, size); @@ -2988,9 +2953,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) return -ENOMEM; } - ret = setup_scratch_page(ggtt->base.dev, - &ggtt->base.scratch_page, - GFP_DMA32); + ret = setup_scratch_page(dev_priv, &ggtt->base.scratch_page, GFP_DMA32); if (ret) { DRM_ERROR("Scratch setup failed\n"); /* iounmap will also get called at remove, but meh */ @@ -3079,7 +3042,7 @@ static void gen6_gmch_remove(struct i915_address_space *vm) struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); iounmap(ggtt->gsm); - cleanup_scratch_page(vm->dev, &vm->scratch_page); + cleanup_scratch_page(to_i915(vm->dev), &vm->scratch_page); } static int gen8_gmch_probe(struct i915_ggtt *ggtt) @@ -3274,11 +3237,13 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv) /* Subtract the guard page before address space initialization to * shrink the range used by drm_mm. */ + mutex_lock(&dev_priv->drm.struct_mutex); ggtt->base.total -= PAGE_SIZE; - i915_address_space_init(&ggtt->base, dev_priv); + i915_address_space_init(&ggtt->base, dev_priv, "[global]"); ggtt->base.total += PAGE_SIZE; if (!HAS_LLC(dev_priv)) ggtt->base.mm.color_adjust = i915_gtt_color_adjust; + mutex_unlock(&dev_priv->drm.struct_mutex); if (!io_mapping_init_wc(&dev_priv->ggtt.mappable, dev_priv->ggtt.mappable_base, @@ -3293,7 +3258,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv) * Initialise stolen early so that we may reserve preallocated * objects for the BIOS to KMS transition. */ - ret = i915_gem_init_stolen(&dev_priv->drm); + ret = i915_gem_init_stolen(dev_priv); if (ret) goto out_gtt_cleanup; @@ -3312,9 +3277,8 @@ int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv) return 0; } -void i915_gem_restore_gtt_mappings(struct drm_device *dev) +void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; struct drm_i915_gem_object *obj, *on; @@ -3327,7 +3291,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) /* clflush objects bound into the GGTT and rebind them. */ list_for_each_entry_safe(obj, on, - &dev_priv->mm.bound_list, global_list) { + &dev_priv->mm.bound_list, global_link) { bool ggtt_bound = false; struct i915_vma *vma; @@ -3349,7 +3313,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) ggtt->base.closed = false; - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv)) chv_setup_private_ppat(dev_priv); else @@ -3358,7 +3322,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) return; } - if (USES_PPGTT(dev)) { + if (USES_PPGTT(dev_priv)) { struct i915_address_space *vm; list_for_each_entry(vm, &dev_priv->vm_list, global_link) { @@ -3379,137 +3343,28 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) i915_ggtt_flush(dev_priv); } -static void -i915_vma_retire(struct i915_gem_active *active, - struct drm_i915_gem_request *rq) -{ - const unsigned int idx = rq->engine->id; - struct i915_vma *vma = - container_of(active, struct i915_vma, last_read[idx]); - - GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); - - i915_vma_clear_active(vma, idx); - if (i915_vma_is_active(vma)) - return; - - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); - if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma))) - WARN_ON(i915_vma_unbind(vma)); -} - -void i915_vma_destroy(struct i915_vma *vma) -{ - GEM_BUG_ON(vma->node.allocated); - GEM_BUG_ON(i915_vma_is_active(vma)); - GEM_BUG_ON(!i915_vma_is_closed(vma)); - GEM_BUG_ON(vma->fence); - - list_del(&vma->vm_link); - if (!i915_vma_is_ggtt(vma)) - i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm)); - - kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma); -} - -void i915_vma_close(struct i915_vma *vma) -{ - GEM_BUG_ON(i915_vma_is_closed(vma)); - vma->flags |= I915_VMA_CLOSED; - - list_del_init(&vma->obj_link); - if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma)) - WARN_ON(i915_vma_unbind(vma)); -} - -static struct i915_vma * -__i915_vma_create(struct drm_i915_gem_object *obj, - struct i915_address_space *vm, - const struct i915_ggtt_view *view) -{ - struct i915_vma *vma; - int i; - - GEM_BUG_ON(vm->closed); - - vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); - if (vma == NULL) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&vma->exec_list); - for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) - init_request_active(&vma->last_read[i], i915_vma_retire); - init_request_active(&vma->last_fence, NULL); - list_add(&vma->vm_link, &vm->unbound_list); - vma->vm = vm; - vma->obj = obj; - vma->size = obj->base.size; - - if (view) { - vma->ggtt_view = *view; - if (view->type == I915_GGTT_VIEW_PARTIAL) { - vma->size = view->params.partial.size; - vma->size <<= PAGE_SHIFT; - } else if (view->type == I915_GGTT_VIEW_ROTATED) { - vma->size = - intel_rotation_info_size(&view->params.rotated); - vma->size <<= PAGE_SHIFT; - } - } - - if (i915_is_ggtt(vm)) { - vma->flags |= I915_VMA_GGTT; - } else { - i915_ppgtt_get(i915_vm_to_ppgtt(vm)); - } - - list_add_tail(&vma->obj_link, &obj->vma_list); - return vma; -} - -static inline bool vma_matches(struct i915_vma *vma, - struct i915_address_space *vm, - const struct i915_ggtt_view *view) -{ - if (vma->vm != vm) - return false; - - if (!i915_vma_is_ggtt(vma)) - return true; - - if (!view) - return vma->ggtt_view.type == 0; - - if (vma->ggtt_view.type != view->type) - return false; - - return memcmp(&vma->ggtt_view.params, - &view->params, - sizeof(view->params)) == 0; -} - -struct i915_vma * -i915_vma_create(struct drm_i915_gem_object *obj, - struct i915_address_space *vm, - const struct i915_ggtt_view *view) -{ - GEM_BUG_ON(view && !i915_is_ggtt(vm)); - GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view)); - - return __i915_vma_create(obj, vm, view); -} - struct i915_vma * i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, struct i915_address_space *vm, const struct i915_ggtt_view *view) { - struct i915_vma *vma; + struct rb_node *rb; + + rb = obj->vma_tree.rb_node; + while (rb) { + struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); + long cmp; - list_for_each_entry_reverse(vma, &obj->vma_list, obj_link) - if (vma_matches(vma, vm, view)) + cmp = i915_vma_compare(vma, vm, view); + if (cmp == 0) return vma; + if (cmp < 0) + rb = rb->rb_right; + else + rb = rb->rb_left; + } + return NULL; } @@ -3520,11 +3375,14 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, { struct i915_vma *vma; + lockdep_assert_held(&obj->base.dev->struct_mutex); GEM_BUG_ON(view && !i915_is_ggtt(vm)); vma = i915_gem_obj_to_vma(obj, vm, view); - if (!vma) - vma = __i915_vma_create(obj, vm, view); + if (!vma) { + vma = i915_vma_create(obj, vm, view); + GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view)); + } GEM_BUG_ON(i915_vma_is_closed(vma)); return vma; @@ -3590,7 +3448,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info, /* Populate source page list from the object. */ i = 0; - for_each_sgt_dma(dma_addr, sgt_iter, obj->pages) + for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages) page_addr_list[i++] = dma_addr; GEM_BUG_ON(i != n_pages); @@ -3626,35 +3484,47 @@ intel_partial_pages(const struct i915_ggtt_view *view, struct drm_i915_gem_object *obj) { struct sg_table *st; - struct scatterlist *sg; - struct sg_page_iter obj_sg_iter; + struct scatterlist *sg, *iter; + unsigned int count = view->params.partial.size; + unsigned int offset; int ret = -ENOMEM; st = kmalloc(sizeof(*st), GFP_KERNEL); if (!st) goto err_st_alloc; - ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); + ret = sg_alloc_table(st, count, GFP_KERNEL); if (ret) goto err_sg_alloc; + iter = i915_gem_object_get_sg(obj, + view->params.partial.offset, + &offset); + GEM_BUG_ON(!iter); + sg = st->sgl; st->nents = 0; - for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, - view->params.partial.offset) - { - if (st->nents >= view->params.partial.size) - break; + do { + unsigned int len; - sg_set_page(sg, NULL, PAGE_SIZE, 0); - sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); - sg_dma_len(sg) = PAGE_SIZE; + len = min(iter->length - (offset << PAGE_SHIFT), + count << PAGE_SHIFT); + sg_set_page(sg, NULL, len, 0); + sg_dma_address(sg) = + sg_dma_address(iter) + (offset << PAGE_SHIFT); + sg_dma_len(sg) = len; - sg = sg_next(sg); st->nents++; - } + count -= len >> PAGE_SHIFT; + if (count == 0) { + sg_mark_end(sg); + return st; + } - return st; + sg = __sg_next(sg); + iter = __sg_next(iter); + offset = 0; + } while (1); err_sg_alloc: kfree(st); @@ -3667,11 +3537,18 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma) { int ret = 0; + /* The vma->pages are only valid within the lifespan of the borrowed + * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so + * must be the vma->pages. A simple rule is that vma->pages must only + * be accessed when the obj->mm.pages are pinned. + */ + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); + if (vma->pages) return 0; if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages; else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) vma->pages = intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj); @@ -3695,94 +3572,3 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma) return ret; } -/** - * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. - * @vma: VMA to map - * @cache_level: mapping cache level - * @flags: flags like global or local mapping - * - * DMA addresses are taken from the scatter-gather table of this object (or of - * this VMA in case of non-default GGTT views) and PTE entries set up. - * Note that DMA addresses are also the only part of the SG table we care about. - */ -int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, - u32 flags) -{ - u32 bind_flags; - u32 vma_flags; - int ret; - - if (WARN_ON(flags == 0)) - return -EINVAL; - - bind_flags = 0; - if (flags & PIN_GLOBAL) - bind_flags |= I915_VMA_GLOBAL_BIND; - if (flags & PIN_USER) - bind_flags |= I915_VMA_LOCAL_BIND; - - vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); - if (flags & PIN_UPDATE) - bind_flags |= vma_flags; - else - bind_flags &= ~vma_flags; - if (bind_flags == 0) - return 0; - - if (vma_flags == 0 && vma->vm->allocate_va_range) { - trace_i915_va_alloc(vma); - ret = vma->vm->allocate_va_range(vma->vm, - vma->node.start, - vma->node.size); - if (ret) - return ret; - } - - ret = vma->vm->bind_vma(vma, cache_level, bind_flags); - if (ret) - return ret; - - vma->flags |= bind_flags; - return 0; -} - -void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) -{ - void __iomem *ptr; - - /* Access through the GTT requires the device to be awake. */ - assert_rpm_wakelock_held(to_i915(vma->vm->dev)); - - lockdep_assert_held(&vma->vm->dev->struct_mutex); - if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) - return IO_ERR_PTR(-ENODEV); - - GEM_BUG_ON(!i915_vma_is_ggtt(vma)); - GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0); - - ptr = vma->iomap; - if (ptr == NULL) { - ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable, - vma->node.start, - vma->node.size); - if (ptr == NULL) - return IO_ERR_PTR(-ENOMEM); - - vma->iomap = ptr; - } - - __i915_vma_pin(vma); - return ptr; -} - -void i915_vma_unpin_and_release(struct i915_vma **p_vma) -{ - struct i915_vma *vma; - - vma = fetch_and_zero(p_vma); - if (!vma) - return; - - i915_vma_unpin(vma); - i915_vma_put(vma); -} diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index c241d8143255..4f35be4c26c7 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -35,7 +35,9 @@ #define __I915_GEM_GTT_H__ #include <linux/io-mapping.h> +#include <linux/mm.h> +#include "i915_gem_timeline.h" #include "i915_gem_request.h" #define I915_FENCE_REG_NONE -1 @@ -118,8 +120,8 @@ typedef uint64_t gen8_ppgtt_pml4e_t; #define GEN8_LEGACY_PDPES 4 #define GEN8_PTES I915_PTES(sizeof(gen8_pte_t)) -#define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\ - GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES) +#define I915_PDPES_PER_PDP(dev_priv) (USES_FULL_48BIT_PPGTT(dev_priv) ?\ + GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES) #define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD) #define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */ @@ -138,6 +140,8 @@ typedef uint64_t gen8_ppgtt_pml4e_t; #define GEN8_PPAT_ELLC_OVERRIDE (0<<2) #define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8)) +struct sg_table; + enum i915_ggtt_view_type { I915_GGTT_VIEW_NORMAL = 0, I915_GGTT_VIEW_ROTATED, @@ -168,133 +172,7 @@ extern const struct i915_ggtt_view i915_ggtt_view_rotated; enum i915_cache_level; -/** - * A VMA represents a GEM BO that is bound into an address space. Therefore, a - * VMA's presence cannot be guaranteed before binding, or after unbinding the - * object into/from the address space. - * - * To make things as simple as possible (ie. no refcounting), a VMA's lifetime - * will always be <= an objects lifetime. So object refcounting should cover us. - */ -struct i915_vma { - struct drm_mm_node node; - struct drm_i915_gem_object *obj; - struct i915_address_space *vm; - struct drm_i915_fence_reg *fence; - struct sg_table *pages; - void __iomem *iomap; - u64 size; - u64 display_alignment; - - unsigned int flags; - /** - * How many users have pinned this object in GTT space. The following - * users can each hold at most one reference: pwrite/pread, execbuffer - * (objects are not allowed multiple times for the same batchbuffer), - * and the framebuffer code. When switching/pageflipping, the - * framebuffer code has at most two buffers pinned per crtc. - * - * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 - * bits with absolutely no headroom. So use 4 bits. - */ -#define I915_VMA_PIN_MASK 0xf -#define I915_VMA_PIN_OVERFLOW BIT(5) - - /** Flags and address space this VMA is bound to */ -#define I915_VMA_GLOBAL_BIND BIT(6) -#define I915_VMA_LOCAL_BIND BIT(7) -#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW) - -#define I915_VMA_GGTT BIT(8) -#define I915_VMA_CAN_FENCE BIT(9) -#define I915_VMA_CLOSED BIT(10) - - unsigned int active; - struct i915_gem_active last_read[I915_NUM_ENGINES]; - struct i915_gem_active last_fence; - - /** - * Support different GGTT views into the same object. - * This means there can be multiple VMA mappings per object and per VM. - * i915_ggtt_view_type is used to distinguish between those entries. - * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also - * assumed in GEM functions which take no ggtt view parameter. - */ - struct i915_ggtt_view ggtt_view; - - /** This object's place on the active/inactive lists */ - struct list_head vm_link; - - struct list_head obj_link; /* Link in the object's VMA list */ - - /** This vma's place in the batchbuffer or on the eviction list */ - struct list_head exec_list; - - /** - * Used for performing relocations during execbuffer insertion. - */ - struct hlist_node exec_node; - unsigned long exec_handle; - struct drm_i915_gem_exec_object2 *exec_entry; -}; - -struct i915_vma * -i915_vma_create(struct drm_i915_gem_object *obj, - struct i915_address_space *vm, - const struct i915_ggtt_view *view); -void i915_vma_unpin_and_release(struct i915_vma **p_vma); - -static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) -{ - return vma->flags & I915_VMA_GGTT; -} - -static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma) -{ - return vma->flags & I915_VMA_CAN_FENCE; -} - -static inline bool i915_vma_is_closed(const struct i915_vma *vma) -{ - return vma->flags & I915_VMA_CLOSED; -} - -static inline unsigned int i915_vma_get_active(const struct i915_vma *vma) -{ - return vma->active; -} - -static inline bool i915_vma_is_active(const struct i915_vma *vma) -{ - return i915_vma_get_active(vma); -} - -static inline void i915_vma_set_active(struct i915_vma *vma, - unsigned int engine) -{ - vma->active |= BIT(engine); -} - -static inline void i915_vma_clear_active(struct i915_vma *vma, - unsigned int engine) -{ - vma->active &= ~BIT(engine); -} - -static inline bool i915_vma_has_active_engine(const struct i915_vma *vma, - unsigned int engine) -{ - return vma->active & BIT(engine); -} - -static inline u32 i915_ggtt_offset(const struct i915_vma *vma) -{ - GEM_BUG_ON(!i915_vma_is_ggtt(vma)); - GEM_BUG_ON(!vma->node.allocated); - GEM_BUG_ON(upper_32_bits(vma->node.start)); - GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1)); - return lower_32_bits(vma->node.start); -} +struct i915_vma; struct i915_page_dma { struct page *page; @@ -341,6 +219,7 @@ struct i915_pml4 { struct i915_address_space { struct drm_mm mm; + struct i915_gem_timeline timeline; struct drm_device *dev; /* Every address space belongs to a struct file - except for the global * GTT that is owned by the driver (and so @file is set to NULL). In @@ -603,16 +482,24 @@ i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n) px_dma(ppgtt->base.scratch_pd); } +static inline struct i915_ggtt * +i915_vm_to_ggtt(struct i915_address_space *vm) +{ + GEM_BUG_ON(!i915_is_ggtt(vm)); + return container_of(vm, struct i915_ggtt, base); +} + int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv); int i915_ggtt_init_hw(struct drm_i915_private *dev_priv); int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv); int i915_gem_init_ggtt(struct drm_i915_private *dev_priv); void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv); -int i915_ppgtt_init_hw(struct drm_device *dev); +int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv); void i915_ppgtt_release(struct kref *kref); struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv, - struct drm_i915_file_private *fpriv); + struct drm_i915_file_private *fpriv, + const char *name); static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt) { if (ppgtt) @@ -625,11 +512,13 @@ static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt) } void i915_check_and_clear_faults(struct drm_i915_private *dev_priv); -void i915_gem_suspend_gtt_mappings(struct drm_device *dev); -void i915_gem_restore_gtt_mappings(struct drm_device *dev); +void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv); +void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv); -int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj); -void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj); +int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages); +void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages); /* Flags used by pin/bind&friends. */ #define PIN_NONBLOCK BIT(0) @@ -647,88 +536,4 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj); #define PIN_OFFSET_FIXED BIT(11) #define PIN_OFFSET_MASK (~4095) -int __i915_vma_do_pin(struct i915_vma *vma, - u64 size, u64 alignment, u64 flags); -static inline int __must_check -i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) -{ - BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW); - BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); - BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); - - /* Pin early to prevent the shrinker/eviction logic from destroying - * our vma as we insert and bind. - */ - if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) - return 0; - - return __i915_vma_do_pin(vma, size, alignment, flags); -} - -static inline int i915_vma_pin_count(const struct i915_vma *vma) -{ - return vma->flags & I915_VMA_PIN_MASK; -} - -static inline bool i915_vma_is_pinned(const struct i915_vma *vma) -{ - return i915_vma_pin_count(vma); -} - -static inline void __i915_vma_pin(struct i915_vma *vma) -{ - vma->flags++; - GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW); -} - -static inline void __i915_vma_unpin(struct i915_vma *vma) -{ - GEM_BUG_ON(!i915_vma_is_pinned(vma)); - vma->flags--; -} - -static inline void i915_vma_unpin(struct i915_vma *vma) -{ - GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - __i915_vma_unpin(vma); -} - -/** - * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture - * @vma: VMA to iomap - * - * The passed in VMA has to be pinned in the global GTT mappable region. - * An extra pinning of the VMA is acquired for the return iomapping, - * the caller must call i915_vma_unpin_iomap to relinquish the pinning - * after the iomapping is no longer required. - * - * Callers must hold the struct_mutex. - * - * Returns a valid iomapped pointer or ERR_PTR. - */ -void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); -#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x)) - -/** - * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap - * @vma: VMA to unpin - * - * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). - * - * Callers must hold the struct_mutex. This function is only valid to be - * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap(). - */ -static inline void i915_vma_unpin_iomap(struct i915_vma *vma) -{ - lockdep_assert_held(&vma->vm->dev->struct_mutex); - GEM_BUG_ON(vma->iomap == NULL); - i915_vma_unpin(vma); -} - -static inline struct page *i915_vma_first_page(struct i915_vma *vma) -{ - GEM_BUG_ON(!vma->pages); - return sg_page(vma->pages->sgl); -} - #endif diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c new file mode 100644 index 000000000000..4b3ff3e5b911 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_internal.c @@ -0,0 +1,170 @@ +/* + * Copyright © 2014-2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include <drm/drmP.h> +#include <drm/i915_drm.h> +#include "i915_drv.h" + +#define QUIET (__GFP_NORETRY | __GFP_NOWARN) + +/* convert swiotlb segment size into sensible units (pages)! */ +#define IO_TLB_SEGPAGES (IO_TLB_SEGSIZE << IO_TLB_SHIFT >> PAGE_SHIFT) + +static void internal_free_pages(struct sg_table *st) +{ + struct scatterlist *sg; + + for (sg = st->sgl; sg; sg = __sg_next(sg)) + __free_pages(sg_page(sg), get_order(sg->length)); + + sg_free_table(st); + kfree(st); +} + +static struct sg_table * +i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + unsigned int npages = obj->base.size / PAGE_SIZE; + struct sg_table *st; + struct scatterlist *sg; + int max_order; + gfp_t gfp; + + st = kmalloc(sizeof(*st), GFP_KERNEL); + if (!st) + return ERR_PTR(-ENOMEM); + + if (sg_alloc_table(st, npages, GFP_KERNEL)) { + kfree(st); + return ERR_PTR(-ENOMEM); + } + + sg = st->sgl; + st->nents = 0; + + max_order = MAX_ORDER; +#ifdef CONFIG_SWIOTLB + if (swiotlb_nr_tbl()) /* minimum max swiotlb size is IO_TLB_SEGSIZE */ + max_order = min(max_order, ilog2(IO_TLB_SEGPAGES)); +#endif + + gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; + if (IS_CRESTLINE(i915) || IS_BROADWATER(i915)) { + /* 965gm cannot relocate objects above 4GiB. */ + gfp &= ~__GFP_HIGHMEM; + gfp |= __GFP_DMA32; + } + + do { + int order = min(fls(npages) - 1, max_order); + struct page *page; + + do { + page = alloc_pages(gfp | (order ? QUIET : 0), order); + if (page) + break; + if (!order--) + goto err; + + /* Limit subsequent allocations as well */ + max_order = order; + } while (1); + + sg_set_page(sg, page, PAGE_SIZE << order, 0); + st->nents++; + + npages -= 1 << order; + if (!npages) { + sg_mark_end(sg); + break; + } + + sg = __sg_next(sg); + } while (1); + + if (i915_gem_gtt_prepare_pages(obj, st)) + goto err; + + /* Mark the pages as dontneed whilst they are still pinned. As soon + * as they are unpinned they are allowed to be reaped by the shrinker, + * and the caller is expected to repopulate - the contents of this + * object are only valid whilst active and pinned. + */ + obj->mm.madv = I915_MADV_DONTNEED; + return st; + +err: + sg_mark_end(sg); + internal_free_pages(st); + return ERR_PTR(-ENOMEM); +} + +static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, + struct sg_table *pages) +{ + i915_gem_gtt_finish_pages(obj, pages); + internal_free_pages(pages); + + obj->mm.dirty = false; + obj->mm.madv = I915_MADV_WILLNEED; +} + +static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { + .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_IS_SHRINKABLE, + .get_pages = i915_gem_object_get_pages_internal, + .put_pages = i915_gem_object_put_pages_internal, +}; + +/** + * Creates a new object that wraps some internal memory for private use. + * This object is not backed by swappable storage, and as such its contents + * are volatile and only valid whilst pinned. If the object is reaped by the + * shrinker, its pages and data will be discarded. Equally, it is not a full + * GEM object and so not valid for access from userspace. This makes it useful + * for hardware interfaces like ringbuffers (which are pinned from the time + * the request is written to the time the hardware stops accessing it), but + * not for contexts (which need to be preserved when not active for later + * reuse). Note that it is not cleared upon allocation. + */ +struct drm_i915_gem_object * +i915_gem_object_create_internal(struct drm_i915_private *i915, + unsigned int size) +{ + struct drm_i915_gem_object *obj; + + obj = i915_gem_object_alloc(&i915->drm); + if (!obj) + return ERR_PTR(-ENOMEM); + + drm_gem_private_object_init(&i915->drm, &obj->base, size); + i915_gem_object_init(obj, &i915_gem_object_internal_ops); + + obj->base.write_domain = I915_GEM_DOMAIN_CPU; + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; + + return obj; +} diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h new file mode 100644 index 000000000000..6a368de9d81e --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_object.h @@ -0,0 +1,338 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#ifndef __I915_GEM_OBJECT_H__ +#define __I915_GEM_OBJECT_H__ + +#include <linux/reservation.h> + +#include <drm/drm_vma_manager.h> +#include <drm/drm_gem.h> +#include <drm/drmP.h> + +#include <drm/i915_drm.h> + +struct drm_i915_gem_object_ops { + unsigned int flags; +#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1 +#define I915_GEM_OBJECT_IS_SHRINKABLE 0x2 + + /* Interface between the GEM object and its backing storage. + * get_pages() is called once prior to the use of the associated set + * of pages before to binding them into the GTT, and put_pages() is + * called after we no longer need them. As we expect there to be + * associated cost with migrating pages between the backing storage + * and making them available for the GPU (e.g. clflush), we may hold + * onto the pages after they are no longer referenced by the GPU + * in case they may be used again shortly (for example migrating the + * pages to a different memory domain within the GTT). put_pages() + * will therefore most likely be called when the object itself is + * being released or under memory pressure (where we attempt to + * reap pages for the shrinker). + */ + struct sg_table *(*get_pages)(struct drm_i915_gem_object *); + void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *); + + int (*dmabuf_export)(struct drm_i915_gem_object *); + void (*release)(struct drm_i915_gem_object *); +}; + +struct drm_i915_gem_object { + struct drm_gem_object base; + + const struct drm_i915_gem_object_ops *ops; + + /** List of VMAs backed by this object */ + struct list_head vma_list; + struct rb_root vma_tree; + + /** Stolen memory for this object, instead of being backed by shmem. */ + struct drm_mm_node *stolen; + struct list_head global_link; + union { + struct rcu_head rcu; + struct llist_node freed; + }; + + /** + * Whether the object is currently in the GGTT mmap. + */ + struct list_head userfault_link; + + /** Used in execbuf to temporarily hold a ref */ + struct list_head obj_exec_link; + + struct list_head batch_pool_link; + + unsigned long flags; + + /** + * Have we taken a reference for the object for incomplete GPU + * activity? + */ +#define I915_BO_ACTIVE_REF 0 + + /* + * Is the object to be mapped as read-only to the GPU + * Only honoured if hardware has relevant pte bit + */ + unsigned long gt_ro:1; + unsigned int cache_level:3; + unsigned int cache_dirty:1; + + atomic_t frontbuffer_bits; + unsigned int frontbuffer_ggtt_origin; /* write once */ + struct i915_gem_active frontbuffer_write; + + /** Current tiling stride for the object, if it's tiled. */ + unsigned int tiling_and_stride; +#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */ +#define TILING_MASK (FENCE_MINIMUM_STRIDE-1) +#define STRIDE_MASK (~TILING_MASK) + + /** Count of VMA actually bound by this object */ + unsigned int bind_count; + unsigned int active_count; + unsigned int pin_display; + + struct { + struct mutex lock; /* protects the pages and their use */ + atomic_t pages_pin_count; + + struct sg_table *pages; + void *mapping; + + struct i915_gem_object_page_iter { + struct scatterlist *sg_pos; + unsigned int sg_idx; /* in pages, but 32bit eek! */ + + struct radix_tree_root radix; + struct mutex lock; /* protects this cache */ + } get_page; + + /** + * Advice: are the backing pages purgeable? + */ + unsigned int madv:2; + + /** + * This is set if the object has been written to since the + * pages were last acquired. + */ + bool dirty:1; + + /** + * This is set if the object has been pinned due to unknown + * swizzling. + */ + bool quirked:1; + } mm; + + /** Breadcrumb of last rendering to the buffer. + * There can only be one writer, but we allow for multiple readers. + * If there is a writer that necessarily implies that all other + * read requests are complete - but we may only be lazily clearing + * the read requests. A read request is naturally the most recent + * request on a ring, so we may have two different write and read + * requests on one ring where the write request is older than the + * read request. This allows for the CPU to read from an active + * buffer by only waiting for the write to complete. + */ + struct reservation_object *resv; + + /** References from framebuffers, locks out tiling changes. */ + unsigned long framebuffer_references; + + /** Record of address bit 17 of each page at last unbind. */ + unsigned long *bit_17; + + struct i915_gem_userptr { + uintptr_t ptr; + unsigned read_only :1; + + struct i915_mm_struct *mm; + struct i915_mmu_object *mmu_object; + struct work_struct *work; + } userptr; + + /** for phys allocated objects */ + struct drm_dma_handle *phys_handle; + + struct reservation_object __builtin_resv; +}; + +static inline struct drm_i915_gem_object * +to_intel_bo(struct drm_gem_object *gem) +{ + /* Assert that to_intel_bo(NULL) == NULL */ + BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base)); + + return container_of(gem, struct drm_i915_gem_object, base); +} + +/** + * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle + * @filp: DRM file private date + * @handle: userspace handle + * + * Returns: + * + * A pointer to the object named by the handle if such exists on @filp, NULL + * otherwise. This object is only valid whilst under the RCU read lock, and + * note carefully the object may be in the process of being destroyed. + */ +static inline struct drm_i915_gem_object * +i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) +{ +#ifdef CONFIG_LOCKDEP + WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); +#endif + return idr_find(&file->object_idr, handle); +} + +static inline struct drm_i915_gem_object * +i915_gem_object_lookup(struct drm_file *file, u32 handle) +{ + struct drm_i915_gem_object *obj; + + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, handle); + if (obj && !kref_get_unless_zero(&obj->base.refcount)) + obj = NULL; + rcu_read_unlock(); + + return obj; +} + +__deprecated +extern struct drm_gem_object * +drm_gem_object_lookup(struct drm_file *file, u32 handle); + +__attribute__((nonnull)) +static inline struct drm_i915_gem_object * +i915_gem_object_get(struct drm_i915_gem_object *obj) +{ + drm_gem_object_reference(&obj->base); + return obj; +} + +__deprecated +extern void drm_gem_object_reference(struct drm_gem_object *); + +__attribute__((nonnull)) +static inline void +i915_gem_object_put(struct drm_i915_gem_object *obj) +{ + __drm_gem_object_unreference(&obj->base); +} + +__deprecated +extern void drm_gem_object_unreference(struct drm_gem_object *); + +__deprecated +extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *); + +static inline bool +i915_gem_object_is_dead(const struct drm_i915_gem_object *obj) +{ + return atomic_read(&obj->base.refcount.refcount) == 0; +} + +static inline bool +i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) +{ + return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; +} + +static inline bool +i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) +{ + return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE; +} + +static inline bool +i915_gem_object_is_active(const struct drm_i915_gem_object *obj) +{ + return obj->active_count; +} + +static inline bool +i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj) +{ + return test_bit(I915_BO_ACTIVE_REF, &obj->flags); +} + +static inline void +i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj) +{ + lockdep_assert_held(&obj->base.dev->struct_mutex); + __set_bit(I915_BO_ACTIVE_REF, &obj->flags); +} + +static inline void +i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj) +{ + lockdep_assert_held(&obj->base.dev->struct_mutex); + __clear_bit(I915_BO_ACTIVE_REF, &obj->flags); +} + +void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj); + +static inline unsigned int +i915_gem_object_get_tiling(struct drm_i915_gem_object *obj) +{ + return obj->tiling_and_stride & TILING_MASK; +} + +static inline bool +i915_gem_object_is_tiled(struct drm_i915_gem_object *obj) +{ + return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; +} + +static inline unsigned int +i915_gem_object_get_stride(struct drm_i915_gem_object *obj) +{ + return obj->tiling_and_stride & STRIDE_MASK; +} + +static inline struct intel_engine_cs * +i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj) +{ + struct intel_engine_cs *engine = NULL; + struct dma_fence *fence; + + rcu_read_lock(); + fence = reservation_object_get_excl_rcu(obj->resv); + rcu_read_unlock(); + + if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence)) + engine = to_request(fence)->engine; + dma_fence_put(fence); + + return engine; +} + +#endif + diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c index a98c0f42badd..5af19b0bf713 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.c +++ b/drivers/gpu/drm/i915/i915_gem_render_state.c @@ -28,17 +28,19 @@ #include "i915_drv.h" #include "intel_renderstate.h" -struct render_state { +struct intel_render_state { const struct intel_renderstate_rodata *rodata; struct i915_vma *vma; - u32 aux_batch_size; - u32 aux_batch_offset; + u32 batch_offset; + u32 batch_size; + u32 aux_offset; + u32 aux_size; }; static const struct intel_renderstate_rodata * -render_state_get_rodata(const struct drm_i915_gem_request *req) +render_state_get_rodata(const struct intel_engine_cs *engine) { - switch (INTEL_GEN(req->i915)) { + switch (INTEL_GEN(engine->i915)) { case 6: return &gen6_null_state; case 7: @@ -63,29 +65,26 @@ render_state_get_rodata(const struct drm_i915_gem_request *req) */ #define OUT_BATCH(batch, i, val) \ do { \ - if (WARN_ON((i) >= PAGE_SIZE / sizeof(u32))) { \ - ret = -ENOSPC; \ - goto err_out; \ - } \ + if ((i) >= PAGE_SIZE / sizeof(u32)) \ + goto err; \ (batch)[(i)++] = (val); \ } while(0) -static int render_state_setup(struct render_state *so) +static int render_state_setup(struct intel_render_state *so, + struct drm_i915_private *i915) { - struct drm_i915_private *dev_priv = to_i915(so->vma->vm->dev); const struct intel_renderstate_rodata *rodata = so->rodata; - const bool has_64bit_reloc = INTEL_GEN(dev_priv) >= 8; + struct drm_i915_gem_object *obj = so->vma->obj; unsigned int i = 0, reloc_index = 0; - struct page *page; + unsigned int needs_clflush; u32 *d; int ret; - ret = i915_gem_object_set_to_cpu_domain(so->vma->obj, true); + ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); if (ret) return ret; - page = i915_gem_object_get_dirty_page(so->vma->obj, 0); - d = kmap(page); + d = kmap_atomic(i915_gem_object_get_dirty_page(obj, 0)); while (i < rodata->batch_items) { u32 s = rodata->batch[i]; @@ -93,12 +92,10 @@ static int render_state_setup(struct render_state *so) if (i * 4 == rodata->reloc[reloc_index]) { u64 r = s + so->vma->node.start; s = lower_32_bits(r); - if (has_64bit_reloc) { + if (HAS_64BIT_RELOC(i915)) { if (i + 1 >= rodata->batch_items || - rodata->batch[i + 1] != 0) { - ret = -EINVAL; - goto err_out; - } + rodata->batch[i + 1] != 0) + goto err; d[i++] = s; s = upper_32_bits(r); @@ -110,12 +107,20 @@ static int render_state_setup(struct render_state *so) d[i++] = s; } + if (rodata->reloc[reloc_index] != -1) { + DRM_ERROR("only %d relocs resolved\n", reloc_index); + goto err; + } + + so->batch_offset = so->vma->node.start; + so->batch_size = rodata->batch_items * sizeof(u32); + while (i % CACHELINE_DWORDS) OUT_BATCH(d, i, MI_NOOP); - so->aux_batch_offset = i * sizeof(u32); + so->aux_offset = i * sizeof(u32); - if (HAS_POOLED_EU(dev_priv)) { + if (HAS_POOLED_EU(i915)) { /* * We always program 3x6 pool config but depending upon which * subslice is disabled HW drops down to appropriate config @@ -143,88 +148,133 @@ static int render_state_setup(struct render_state *so) } OUT_BATCH(d, i, MI_BATCH_BUFFER_END); - so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset; - + so->aux_size = i * sizeof(u32) - so->aux_offset; + so->aux_offset += so->batch_offset; /* * Since we are sending length, we need to strictly conform to * all requirements. For Gen2 this must be a multiple of 8. */ - so->aux_batch_size = ALIGN(so->aux_batch_size, 8); - - kunmap(page); - - ret = i915_gem_object_set_to_gtt_domain(so->vma->obj, false); - if (ret) - return ret; - - if (rodata->reloc[reloc_index] != -1) { - DRM_ERROR("only %d relocs resolved\n", reloc_index); - return -EINVAL; - } + so->aux_size = ALIGN(so->aux_size, 8); - return 0; + if (needs_clflush) + drm_clflush_virt_range(d, i * sizeof(u32)); + kunmap_atomic(d); -err_out: - kunmap(page); + ret = i915_gem_object_set_to_gtt_domain(obj, false); +out: + i915_gem_obj_finish_shmem_access(obj); return ret; + +err: + kunmap_atomic(d); + ret = -EINVAL; + goto out; } #undef OUT_BATCH -int i915_gem_render_state_init(struct drm_i915_gem_request *req) +int i915_gem_render_state_init(struct intel_engine_cs *engine) { - struct render_state so; + struct intel_render_state *so; + const struct intel_renderstate_rodata *rodata; struct drm_i915_gem_object *obj; int ret; - if (WARN_ON(req->engine->id != RCS)) - return -ENOENT; + if (engine->id != RCS) + return 0; - so.rodata = render_state_get_rodata(req); - if (!so.rodata) + rodata = render_state_get_rodata(engine); + if (!rodata) return 0; - if (so.rodata->batch_items * 4 > 4096) + if (rodata->batch_items * 4 > 4096) return -EINVAL; - obj = i915_gem_object_create(&req->i915->drm, 4096); - if (IS_ERR(obj)) - return PTR_ERR(obj); + so = kmalloc(sizeof(*so), GFP_KERNEL); + if (!so) + return -ENOMEM; - so.vma = i915_vma_create(obj, &req->i915->ggtt.base, NULL); - if (IS_ERR(so.vma)) { - ret = PTR_ERR(so.vma); - goto err_obj; + obj = i915_gem_object_create_internal(engine->i915, 4096); + if (IS_ERR(obj)) { + ret = PTR_ERR(obj); + goto err_free; } - ret = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL); - if (ret) + so->vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL); + if (IS_ERR(so->vma)) { + ret = PTR_ERR(so->vma); goto err_obj; + } + + so->rodata = rodata; + engine->render_state = so; + return 0; + +err_obj: + i915_gem_object_put(obj); +err_free: + kfree(so); + return ret; +} + +int i915_gem_render_state_emit(struct drm_i915_gem_request *req) +{ + struct intel_render_state *so; + int ret; + + lockdep_assert_held(&req->i915->drm.struct_mutex); - ret = render_state_setup(&so); + so = req->engine->render_state; + if (!so) + return 0; + + /* Recreate the page after shrinking */ + if (!so->vma->obj->mm.pages) + so->batch_offset = -1; + + ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH); if (ret) - goto err_unpin; + return ret; + + if (so->vma->node.start != so->batch_offset) { + ret = render_state_setup(so, req->i915); + if (ret) + goto err_unpin; + } - ret = req->engine->emit_bb_start(req, so.vma->node.start, - so.rodata->batch_items * 4, + ret = req->engine->emit_bb_start(req, + so->batch_offset, so->batch_size, I915_DISPATCH_SECURE); if (ret) goto err_unpin; - if (so.aux_batch_size > 8) { + if (so->aux_size > 8) { ret = req->engine->emit_bb_start(req, - (so.vma->node.start + - so.aux_batch_offset), - so.aux_batch_size, + so->aux_offset, so->aux_size, I915_DISPATCH_SECURE); if (ret) goto err_unpin; } - i915_vma_move_to_active(so.vma, req, 0); + i915_vma_move_to_active(so->vma, req, 0); err_unpin: - i915_vma_unpin(so.vma); -err_obj: - i915_gem_object_put(obj); + i915_vma_unpin(so->vma); return ret; } + +void i915_gem_render_state_fini(struct intel_engine_cs *engine) +{ + struct intel_render_state *so; + struct drm_i915_gem_object *obj; + + so = fetch_and_zero(&engine->render_state); + if (!so) + return; + + obj = so->vma->obj; + + i915_vma_close(so->vma); + __i915_gem_object_release_unless_active(obj); + + kfree(so); +} diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h index 18cce3f06e9c..87481845799d 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.h +++ b/drivers/gpu/drm/i915/i915_gem_render_state.h @@ -26,6 +26,8 @@ struct drm_i915_gem_request; -int i915_gem_render_state_init(struct drm_i915_gem_request *req); +int i915_gem_render_state_init(struct intel_engine_cs *engine); +int i915_gem_render_state_emit(struct drm_i915_gem_request *req); +void i915_gem_render_state_fini(struct intel_engine_cs *engine); #endif /* _I915_GEM_RENDER_STATE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index f9af2a00625e..57194471f8cd 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -23,6 +23,7 @@ */ #include <linux/prefetch.h> +#include <linux/dma-fence-array.h> #include "i915_drv.h" @@ -33,13 +34,7 @@ static const char *i915_fence_get_driver_name(struct dma_fence *fence) static const char *i915_fence_get_timeline_name(struct dma_fence *fence) { - /* Timelines are bound by eviction to a VM. However, since - * we only have a global seqno at the moment, we only have - * a single timeline. Note that each timeline will have - * multiple execution contexts (fence contexts) as we allow - * engines within a single timeline to execute in parallel. - */ - return "global"; + return to_request(fence)->timeline->common->name; } static bool i915_fence_signaled(struct dma_fence *fence) @@ -58,43 +53,9 @@ static bool i915_fence_enable_signaling(struct dma_fence *fence) static signed long i915_fence_wait(struct dma_fence *fence, bool interruptible, - signed long timeout_jiffies) + signed long timeout) { - s64 timeout_ns, *timeout; - int ret; - - if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT) { - timeout_ns = jiffies_to_nsecs(timeout_jiffies); - timeout = &timeout_ns; - } else { - timeout = NULL; - } - - ret = i915_wait_request(to_request(fence), - interruptible, timeout, - NO_WAITBOOST); - if (ret == -ETIME) - return 0; - - if (ret < 0) - return ret; - - if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT) - timeout_jiffies = nsecs_to_jiffies(timeout_ns); - - return timeout_jiffies; -} - -static void i915_fence_value_str(struct dma_fence *fence, char *str, int size) -{ - snprintf(str, size, "%u", fence->seqno); -} - -static void i915_fence_timeline_value_str(struct dma_fence *fence, char *str, - int size) -{ - snprintf(str, size, "%u", - intel_engine_get_seqno(to_request(fence)->engine)); + return i915_wait_request(to_request(fence), interruptible, timeout); } static void i915_fence_release(struct dma_fence *fence) @@ -111,8 +72,6 @@ const struct dma_fence_ops i915_fence_ops = { .signaled = i915_fence_signaled, .wait = i915_fence_wait, .release = i915_fence_release, - .fence_value_str = i915_fence_value_str, - .timeline_value_str = i915_fence_timeline_value_str, }; int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, @@ -154,6 +113,82 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) spin_unlock(&file_priv->mm.lock); } +static struct i915_dependency * +i915_dependency_alloc(struct drm_i915_private *i915) +{ + return kmem_cache_alloc(i915->dependencies, GFP_KERNEL); +} + +static void +i915_dependency_free(struct drm_i915_private *i915, + struct i915_dependency *dep) +{ + kmem_cache_free(i915->dependencies, dep); +} + +static void +__i915_priotree_add_dependency(struct i915_priotree *pt, + struct i915_priotree *signal, + struct i915_dependency *dep, + unsigned long flags) +{ + INIT_LIST_HEAD(&dep->dfs_link); + list_add(&dep->wait_link, &signal->waiters_list); + list_add(&dep->signal_link, &pt->signalers_list); + dep->signaler = signal; + dep->flags = flags; +} + +static int +i915_priotree_add_dependency(struct drm_i915_private *i915, + struct i915_priotree *pt, + struct i915_priotree *signal) +{ + struct i915_dependency *dep; + + dep = i915_dependency_alloc(i915); + if (!dep) + return -ENOMEM; + + __i915_priotree_add_dependency(pt, signal, dep, I915_DEPENDENCY_ALLOC); + return 0; +} + +static void +i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt) +{ + struct i915_dependency *dep, *next; + + GEM_BUG_ON(!RB_EMPTY_NODE(&pt->node)); + + /* Everyone we depended upon (the fences we wait to be signaled) + * should retire before us and remove themselves from our list. + * However, retirement is run independently on each timeline and + * so we may be called out-of-order. + */ + list_for_each_entry_safe(dep, next, &pt->signalers_list, signal_link) { + list_del(&dep->wait_link); + if (dep->flags & I915_DEPENDENCY_ALLOC) + i915_dependency_free(i915, dep); + } + + /* Remove ourselves from everyone who depends upon us */ + list_for_each_entry_safe(dep, next, &pt->waiters_list, wait_link) { + list_del(&dep->signal_link); + if (dep->flags & I915_DEPENDENCY_ALLOC) + i915_dependency_free(i915, dep); + } +} + +static void +i915_priotree_init(struct i915_priotree *pt) +{ + INIT_LIST_HEAD(&pt->signalers_list); + INIT_LIST_HEAD(&pt->waiters_list); + RB_CLEAR_NODE(&pt->node); + pt->priority = INT_MIN; +} + void i915_gem_retire_noop(struct i915_gem_active *active, struct drm_i915_gem_request *request) { @@ -164,8 +199,17 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) { struct i915_gem_active *active, *next; + lockdep_assert_held(&request->i915->drm.struct_mutex); + GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit)); + GEM_BUG_ON(!i915_sw_fence_signaled(&request->execute)); + GEM_BUG_ON(!i915_gem_request_completed(request)); + GEM_BUG_ON(!request->i915->gt.active_requests); + trace_i915_gem_request_retire(request); - list_del(&request->link); + + spin_lock_irq(&request->engine->timeline->lock); + list_del_init(&request->link); + spin_unlock_irq(&request->engine->timeline->lock); /* We know the GPU must have read the request to have * sent us the seqno + interrupt, so use the position @@ -177,6 +221,12 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) */ list_del(&request->ring_link); request->ring->last_retired_head = request->postfix; + if (!--request->i915->gt.active_requests) { + GEM_BUG_ON(!request->i915->gt.awake); + mod_delayed_work(request->i915->wq, + &request->i915->gt.idle_work, + msecs_to_jiffies(100)); + } /* Walk through the active list, calling retire on each. This allows * objects to track their GPU activity and mark themselves as idle @@ -214,6 +264,10 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) } i915_gem_context_put(request->ctx); + + dma_fence_signal(&request->fence); + + i915_priotree_fini(request->i915, &request->priotree); i915_gem_request_put(request); } @@ -223,10 +277,11 @@ void i915_gem_request_retire_upto(struct drm_i915_gem_request *req) struct drm_i915_gem_request *tmp; lockdep_assert_held(&req->i915->drm.struct_mutex); - GEM_BUG_ON(list_empty(&req->link)); + if (list_empty(&req->link)) + return; do { - tmp = list_first_entry(&engine->request_list, + tmp = list_first_entry(&engine->timeline->requests, typeof(*tmp), link); i915_gem_request_retire(tmp); @@ -253,40 +308,50 @@ static int i915_gem_check_wedge(struct drm_i915_private *dev_priv) return 0; } -static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno) +static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno) { + struct i915_gem_timeline *timeline = &i915->gt.global_timeline; struct intel_engine_cs *engine; enum intel_engine_id id; int ret; /* Carefully retire all requests without writing to the rings */ - for_each_engine(engine, dev_priv, id) { - ret = intel_engine_idle(engine, - I915_WAIT_INTERRUPTIBLE | - I915_WAIT_LOCKED); - if (ret) - return ret; - } - i915_gem_retire_requests(dev_priv); + ret = i915_gem_wait_for_idle(i915, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED); + if (ret) + return ret; + + i915_gem_retire_requests(i915); + GEM_BUG_ON(i915->gt.active_requests > 1); /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ - if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) { - while (intel_kick_waiters(dev_priv) || - intel_kick_signalers(dev_priv)) - yield(); + if (!i915_seqno_passed(seqno, atomic_read(&timeline->next_seqno))) { + while (intel_breadcrumbs_busy(i915)) + cond_resched(); /* spin until threads are complete */ } + atomic_set(&timeline->next_seqno, seqno); /* Finally reset hw state */ - for_each_engine(engine, dev_priv, id) - intel_engine_init_seqno(engine, seqno); + for_each_engine(engine, i915, id) + intel_engine_init_global_seqno(engine, seqno); + + list_for_each_entry(timeline, &i915->gt.timelines, link) { + for_each_engine(engine, i915, id) { + struct intel_timeline *tl = &timeline->engine[id]; + + memset(tl->sync_seqno, 0, sizeof(tl->sync_seqno)); + } + } return 0; } -int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) +int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno) { struct drm_i915_private *dev_priv = to_i915(dev); - int ret; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); if (seqno == 0) return -EINVAL; @@ -294,29 +359,87 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) /* HWS page needs to be set less than what we * will inject to ring */ - ret = i915_gem_init_seqno(dev_priv, seqno - 1); - if (ret) + return i915_gem_init_global_seqno(dev_priv, seqno - 1); +} + +static int reserve_global_seqno(struct drm_i915_private *i915) +{ + u32 active_requests = ++i915->gt.active_requests; + u32 next_seqno = atomic_read(&i915->gt.global_timeline.next_seqno); + int ret; + + /* Reservation is fine until we need to wrap around */ + if (likely(next_seqno + active_requests > next_seqno)) + return 0; + + ret = i915_gem_init_global_seqno(i915, 0); + if (ret) { + i915->gt.active_requests--; return ret; + } - dev_priv->next_seqno = seqno; return 0; } -static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno) +static u32 __timeline_get_seqno(struct i915_gem_timeline *tl) { - /* reserve 0 for non-seqno */ - if (unlikely(dev_priv->next_seqno == 0)) { - int ret; + /* next_seqno only incremented under a mutex */ + return ++tl->next_seqno.counter; +} - ret = i915_gem_init_seqno(dev_priv, 0); - if (ret) - return ret; +static u32 timeline_get_seqno(struct i915_gem_timeline *tl) +{ + return atomic_inc_return(&tl->next_seqno); +} - dev_priv->next_seqno = 1; - } +void __i915_gem_request_submit(struct drm_i915_gem_request *request) +{ + struct intel_engine_cs *engine = request->engine; + struct intel_timeline *timeline; + u32 seqno; - *seqno = dev_priv->next_seqno++; - return 0; + /* Transfer from per-context onto the global per-engine timeline */ + timeline = engine->timeline; + GEM_BUG_ON(timeline == request->timeline); + assert_spin_locked(&timeline->lock); + + seqno = timeline_get_seqno(timeline->common); + GEM_BUG_ON(!seqno); + GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno)); + + GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno, seqno)); + request->previous_seqno = timeline->last_submitted_seqno; + timeline->last_submitted_seqno = seqno; + + /* We may be recursing from the signal callback of another i915 fence */ + spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); + request->global_seqno = seqno; + if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) + intel_engine_enable_signaling(request); + spin_unlock(&request->lock); + + GEM_BUG_ON(!request->global_seqno); + engine->emit_breadcrumb(request, + request->ring->vaddr + request->postfix); + + spin_lock(&request->timeline->lock); + list_move_tail(&request->link, &timeline->requests); + spin_unlock(&request->timeline->lock); + + i915_sw_fence_commit(&request->execute); +} + +void i915_gem_request_submit(struct drm_i915_gem_request *request) +{ + struct intel_engine_cs *engine = request->engine; + unsigned long flags; + + /* Will be called from irq-context when using foreign fences. */ + spin_lock_irqsave(&engine->timeline->lock, flags); + + __i915_gem_request_submit(request); + + spin_unlock_irqrestore(&engine->timeline->lock, flags); } static int __i915_sw_fence_call @@ -325,15 +448,31 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) struct drm_i915_gem_request *request = container_of(fence, typeof(*request), submit); - /* Will be called from irq-context when using foreign DMA fences */ - switch (state) { case FENCE_COMPLETE: - request->engine->last_submitted_seqno = request->fence.seqno; request->engine->submit_request(request); break; case FENCE_FREE: + i915_gem_request_put(request); + break; + } + + return NOTIFY_DONE; +} + +static int __i915_sw_fence_call +execute_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) +{ + struct drm_i915_gem_request *request = + container_of(fence, typeof(*request), execute); + + switch (state) { + case FENCE_COMPLETE: + break; + + case FENCE_FREE: + i915_gem_request_put(request); break; } @@ -358,9 +497,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, { struct drm_i915_private *dev_priv = engine->i915; struct drm_i915_gem_request *req; - u32 seqno; int ret; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex * and restart. @@ -369,10 +509,14 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, if (ret) return ERR_PTR(ret); + ret = reserve_global_seqno(dev_priv); + if (ret) + return ERR_PTR(ret); + /* Move the oldest request to the slab-cache (if not in use!) */ - req = list_first_entry_or_null(&engine->request_list, + req = list_first_entry_or_null(&engine->timeline->requests, typeof(*req), link); - if (req && i915_gem_request_completed(req)) + if (req && __i915_gem_request_completed(req)) i915_gem_request_retire(req); /* Beware: Dragons be flying overhead. @@ -383,7 +527,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * of being read by __i915_gem_active_get_rcu(). As such, * we have to be very careful when overwriting the contents. During * the RCU lookup, we change chase the request->engine pointer, - * read the request->fence.seqno and increment the reference count. + * read the request->global_seqno and increment the reference count. * * The reference count is incremented atomically. If it is zero, * the lookup knows the request is unallocated and complete. Otherwise, @@ -404,21 +548,32 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * Do not use kmem_cache_zalloc() here! */ req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL); - if (!req) - return ERR_PTR(-ENOMEM); + if (!req) { + ret = -ENOMEM; + goto err_unreserve; + } - ret = i915_gem_get_seqno(dev_priv, &seqno); - if (ret) - goto err; + req->timeline = i915_gem_context_lookup_timeline(ctx, engine); + GEM_BUG_ON(req->timeline == engine->timeline); spin_lock_init(&req->lock); dma_fence_init(&req->fence, &i915_fence_ops, &req->lock, - engine->fence_context, - seqno); + req->timeline->fence_context, + __timeline_get_seqno(req->timeline->common)); - i915_sw_fence_init(&req->submit, submit_notify); + /* We bump the ref for the fence chain */ + i915_sw_fence_init(&i915_gem_request_get(req)->submit, submit_notify); + i915_sw_fence_init(&i915_gem_request_get(req)->execute, execute_notify); + + /* Ensure that the execute fence completes after the submit fence - + * as we complete the execute fence from within the submit fence + * callback, its completion would otherwise be visible first. + */ + i915_sw_fence_await_sw_fence(&req->execute, &req->submit, &req->execq); + + i915_priotree_init(&req->priotree); INIT_LIST_HEAD(&req->active_list); req->i915 = dev_priv; @@ -426,6 +581,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, req->ctx = i915_gem_context_get(ctx); /* No zalloc, must clear what we need by hand */ + req->global_seqno = 0; req->previous_context = NULL; req->file_priv = NULL; req->batch = NULL; @@ -438,6 +594,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * away, e.g. because a GPU scheduler has deferred it. */ req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST; + GEM_BUG_ON(req->reserved_space < engine->emit_breadcrumb_sz); if (i915.enable_execlists) ret = intel_logical_ring_alloc_request_extras(req); @@ -457,8 +614,9 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, err_ctx: i915_gem_context_put(ctx); -err: kmem_cache_free(dev_priv->requests, req); +err_unreserve: + dev_priv->gt.active_requests--; return ERR_PTR(ret); } @@ -466,15 +624,36 @@ static int i915_gem_request_await_request(struct drm_i915_gem_request *to, struct drm_i915_gem_request *from) { - int idx, ret; + int ret; GEM_BUG_ON(to == from); - if (to->engine == from->engine) + if (to->engine->schedule) { + ret = i915_priotree_add_dependency(to->i915, + &to->priotree, + &from->priotree); + if (ret < 0) + return ret; + } + + if (to->timeline == from->timeline) return 0; - idx = intel_engine_sync_index(from->engine, to->engine); - if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx]) + if (to->engine == from->engine) { + ret = i915_sw_fence_await_sw_fence_gfp(&to->submit, + &from->submit, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + + if (!from->global_seqno) { + ret = i915_sw_fence_await_dma_fence(&to->submit, + &from->fence, 0, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + + if (from->global_seqno <= to->timeline->sync_seqno[from->engine->id]) return 0; trace_i915_gem_ring_sync_to(to, from); @@ -492,7 +671,54 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret; } - from->engine->semaphore.sync_seqno[idx] = from->fence.seqno; + to->timeline->sync_seqno[from->engine->id] = from->global_seqno; + return 0; +} + +int +i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req, + struct dma_fence *fence) +{ + struct dma_fence_array *array; + int ret; + int i; + + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return 0; + + if (dma_fence_is_i915(fence)) + return i915_gem_request_await_request(req, to_request(fence)); + + if (!dma_fence_is_array(fence)) { + ret = i915_sw_fence_await_dma_fence(&req->submit, + fence, I915_FENCE_TIMEOUT, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + + /* Note that if the fence-array was created in signal-on-any mode, + * we should *not* decompose it into its individual fences. However, + * we don't currently store which mode the fence-array is operating + * in. Fortunately, the only user of signal-on-any is private to + * amdgpu and we should not see any incoming fence-array from + * sync-file being in signal-on-any mode. + */ + + array = to_dma_fence_array(fence); + for (i = 0; i < array->num_fences; i++) { + struct dma_fence *child = array->fences[i]; + + if (dma_fence_is_i915(child)) + ret = i915_gem_request_await_request(req, + to_request(child)); + else + ret = i915_sw_fence_await_dma_fence(&req->submit, + child, I915_FENCE_TIMEOUT, + GFP_KERNEL); + if (ret < 0) + return ret; + } + return 0; } @@ -521,43 +747,52 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to, struct drm_i915_gem_object *obj, bool write) { - struct i915_gem_active *active; - unsigned long active_mask; - int idx; + struct dma_fence *excl; + int ret = 0; if (write) { - active_mask = i915_gem_object_get_active(obj); - active = obj->last_read; + struct dma_fence **shared; + unsigned int count, i; + + ret = reservation_object_get_fences_rcu(obj->resv, + &excl, &count, &shared); + if (ret) + return ret; + + for (i = 0; i < count; i++) { + ret = i915_gem_request_await_dma_fence(to, shared[i]); + if (ret) + break; + + dma_fence_put(shared[i]); + } + + for (; i < count; i++) + dma_fence_put(shared[i]); + kfree(shared); } else { - active_mask = 1; - active = &obj->last_write; + excl = reservation_object_get_excl_rcu(obj->resv); } - for_each_active(active_mask, idx) { - struct drm_i915_gem_request *request; - int ret; - - request = i915_gem_active_peek(&active[idx], - &obj->base.dev->struct_mutex); - if (!request) - continue; + if (excl) { + if (ret == 0) + ret = i915_gem_request_await_dma_fence(to, excl); - ret = i915_gem_request_await_request(to, request); - if (ret) - return ret; + dma_fence_put(excl); } - return 0; + return ret; } static void i915_gem_mark_busy(const struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; - dev_priv->gt.active_engines |= intel_engine_flag(engine); if (dev_priv->gt.awake) return; + GEM_BUG_ON(!dev_priv->gt.active_requests); + intel_runtime_pm_get_noresume(dev_priv); dev_priv->gt.awake = true; @@ -580,11 +815,11 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) { struct intel_engine_cs *engine = request->engine; struct intel_ring *ring = request->ring; + struct intel_timeline *timeline = request->timeline; struct drm_i915_gem_request *prev; - u32 request_start; - u32 reserved_tail; - int ret; + int err; + lockdep_assert_held(&request->i915->drm.struct_mutex); trace_i915_gem_request_add(request); /* @@ -592,8 +827,6 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * should already have been reserved in the ring buffer. Let the ring * know that it is time to use that space up. */ - request_start = ring->tail; - reserved_tail = request->reserved_space; request->reserved_space = 0; /* @@ -604,10 +837,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * what. */ if (flush_caches) { - ret = engine->emit_flush(request, EMIT_FLUSH); + err = engine->emit_flush(request, EMIT_FLUSH); /* Not allowed to fail! */ - WARN(ret, "engine->emit_flush() failed: %d!\n", ret); + WARN(err, "engine->emit_flush() failed: %d!\n", err); } /* Record the position of the start of the breadcrumb so that @@ -615,20 +848,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * GPU processing the request, we never over-estimate the * position of the ring's HEAD. */ + err = intel_ring_begin(request, engine->emit_breadcrumb_sz); + GEM_BUG_ON(err); request->postfix = ring->tail; - - /* Not allowed to fail! */ - ret = engine->emit_request(request); - WARN(ret, "(%s)->emit_request failed: %d!\n", engine->name, ret); - - /* Sanity check that the reserved size was large enough. */ - ret = ring->tail - request_start; - if (ret < 0) - ret += ring->size; - WARN_ONCE(ret > reserved_tail, - "Not enough space reserved (%d bytes) " - "for adding the request (%d bytes)\n", - reserved_tail, ret); + ring->tail += engine->emit_breadcrumb_sz * sizeof(u32); /* Seal the request and mark it as pending execution. Note that * we may inspect this state, without holding any locks, during @@ -636,21 +859,46 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * see a more recent value in the hws than we are tracking. */ - prev = i915_gem_active_raw(&engine->last_request, + prev = i915_gem_active_raw(&timeline->last_request, &request->i915->drm.struct_mutex); - if (prev) + if (prev) { i915_sw_fence_await_sw_fence(&request->submit, &prev->submit, &request->submitq); + if (engine->schedule) + __i915_priotree_add_dependency(&request->priotree, + &prev->priotree, + &request->dep, + 0); + } + + spin_lock_irq(&timeline->lock); + list_add_tail(&request->link, &timeline->requests); + spin_unlock_irq(&timeline->lock); + + GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno, + request->fence.seqno)); + + timeline->last_submitted_seqno = request->fence.seqno; + i915_gem_active_set(&timeline->last_request, request); - request->emitted_jiffies = jiffies; - request->previous_seqno = engine->last_pending_seqno; - engine->last_pending_seqno = request->fence.seqno; - i915_gem_active_set(&engine->last_request, request); - list_add_tail(&request->link, &engine->request_list); list_add_tail(&request->ring_link, &ring->request_list); + request->emitted_jiffies = jiffies; i915_gem_mark_busy(engine); + /* Let the backend know a new request has arrived that may need + * to adjust the existing execution schedule due to a high priority + * request - i.e. we may want to preempt the current request in order + * to run a high priority dependency chain *before* we can execute this + * request. + * + * This is called before the request is ready to run so that we can + * decide whether to preempt the entire chain so that it is ready to + * run at the earliest possible convenience. + */ + if (engine->schedule) + engine->schedule(request, request->ctx->priority); + local_bh_disable(); i915_sw_fence_commit(&request->submit); local_bh_enable(); /* Kick the execlists tasklet if just scheduled */ @@ -715,7 +963,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req, timeout_us += local_clock_us(&cpu); do { - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; if (signal_pending_state(state, current)) @@ -730,76 +978,102 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req, return false; } +static long +__i915_request_wait_for_execute(struct drm_i915_gem_request *request, + unsigned int flags, + long timeout) +{ + const int state = flags & I915_WAIT_INTERRUPTIBLE ? + TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; + wait_queue_head_t *q = &request->i915->gpu_error.wait_queue; + DEFINE_WAIT(reset); + DEFINE_WAIT(wait); + + if (flags & I915_WAIT_LOCKED) + add_wait_queue(q, &reset); + + do { + prepare_to_wait(&request->execute.wait, &wait, state); + + if (i915_sw_fence_done(&request->execute)) + break; + + if (flags & I915_WAIT_LOCKED && + i915_reset_in_progress(&request->i915->gpu_error)) { + __set_current_state(TASK_RUNNING); + i915_reset(request->i915); + reset_wait_queue(q, &reset); + continue; + } + + if (signal_pending_state(state, current)) { + timeout = -ERESTARTSYS; + break; + } + + timeout = io_schedule_timeout(timeout); + } while (timeout); + finish_wait(&request->execute.wait, &wait); + + if (flags & I915_WAIT_LOCKED) + remove_wait_queue(q, &reset); + + return timeout; +} + /** * i915_wait_request - wait until execution of request has finished - * @req: duh! + * @req: the request to wait upon * @flags: how to wait - * @timeout: in - how long to wait (NULL forever); out - how much time remaining - * @rps: client to charge for RPS boosting + * @timeout: how long to wait in jiffies + * + * i915_wait_request() waits for the request to be completed, for a + * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an + * unbounded wait). * - * Note: It is of utmost importance that the passed in seqno and reset_counter - * values have been read by the caller in an smp safe manner. Where read-side - * locks are involved, it is sufficient to read the reset_counter before - * unlocking the lock that protects the seqno. For lockless tricks, the - * reset_counter _must_ be read before, and an appropriate smp_rmb must be - * inserted. + * If the caller holds the struct_mutex, the caller must pass I915_WAIT_LOCKED + * in via the flags, and vice versa if the struct_mutex is not held, the caller + * must not specify that the wait is locked. * - * Returns 0 if the request was found within the alloted time. Else returns the - * errno with remaining time filled in timeout argument. + * Returns the remaining time (in jiffies) if the request completed, which may + * be zero or -ETIME if the request is unfinished after the timeout expires. + * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is + * pending before the request completes. */ -int i915_wait_request(struct drm_i915_gem_request *req, - unsigned int flags, - s64 *timeout, - struct intel_rps_client *rps) +long i915_wait_request(struct drm_i915_gem_request *req, + unsigned int flags, + long timeout) { const int state = flags & I915_WAIT_INTERRUPTIBLE ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; DEFINE_WAIT(reset); struct intel_wait wait; - unsigned long timeout_remain; - int ret = 0; might_sleep(); #if IS_ENABLED(CONFIG_LOCKDEP) - GEM_BUG_ON(!!lockdep_is_held(&req->i915->drm.struct_mutex) != + GEM_BUG_ON(debug_locks && + !!lockdep_is_held(&req->i915->drm.struct_mutex) != !!(flags & I915_WAIT_LOCKED)); #endif + GEM_BUG_ON(timeout < 0); if (i915_gem_request_completed(req)) - return 0; - - timeout_remain = MAX_SCHEDULE_TIMEOUT; - if (timeout) { - if (WARN_ON(*timeout < 0)) - return -EINVAL; + return timeout; - if (*timeout == 0) - return -ETIME; - - /* Record current time in case interrupted, or wedged */ - timeout_remain = nsecs_to_jiffies_timeout(*timeout); - *timeout += ktime_get_raw_ns(); - } + if (!timeout) + return -ETIME; trace_i915_gem_request_wait_begin(req); - /* This client is about to stall waiting for the GPU. In many cases - * this is undesirable and limits the throughput of the system, as - * many clients cannot continue processing user input/output whilst - * blocked. RPS autotuning may take tens of milliseconds to respond - * to the GPU load and thus incurs additional latency for the client. - * We can circumvent that by promoting the GPU frequency to maximum - * before we wait. This makes the GPU throttle up much more quickly - * (good for benchmarks and user experience, e.g. window animations), - * but at a cost of spending more power processing the workload - * (bad for battery). Not all clients even want their results - * immediately and for them we should just let the GPU select its own - * frequency to maximise efficiency. To prevent a single client from - * forcing the clocks too high for the whole system, we only allow - * each client to waitboost once in a busy period. - */ - if (IS_RPS_CLIENT(rps) && INTEL_GEN(req->i915) >= 6) - gen6_rps_boost(req->i915, rps, req->emitted_jiffies); + if (!i915_sw_fence_done(&req->execute)) { + timeout = __i915_request_wait_for_execute(req, flags, timeout); + if (timeout < 0) + goto complete; + + GEM_BUG_ON(!i915_sw_fence_done(&req->execute)); + } + GEM_BUG_ON(!i915_sw_fence_done(&req->submit)); + GEM_BUG_ON(!req->global_seqno); /* Optimistic short spin before touching IRQs */ if (i915_spin_request(req, state, 5)) @@ -809,7 +1083,7 @@ int i915_wait_request(struct drm_i915_gem_request *req, if (flags & I915_WAIT_LOCKED) add_wait_queue(&req->i915->gpu_error.wait_queue, &reset); - intel_wait_init(&wait, req->fence.seqno); + intel_wait_init(&wait, req->global_seqno); if (intel_engine_add_wait(req->engine, &wait)) /* In order to check that we haven't missed the interrupt * as we enabled it, we need to kick ourselves to do a @@ -819,16 +1093,17 @@ int i915_wait_request(struct drm_i915_gem_request *req, for (;;) { if (signal_pending_state(state, current)) { - ret = -ERESTARTSYS; + timeout = -ERESTARTSYS; break; } - timeout_remain = io_schedule_timeout(timeout_remain); - if (timeout_remain == 0) { - ret = -ETIME; + if (!timeout) { + timeout = -ETIME; break; } + timeout = io_schedule_timeout(timeout); + if (intel_wait_complete(&wait)) break; @@ -875,74 +1150,32 @@ wakeup: complete: trace_i915_gem_request_wait_end(req); - if (timeout) { - *timeout -= ktime_get_raw_ns(); - if (*timeout < 0) - *timeout = 0; - - /* - * Apparently ktime isn't accurate enough and occasionally has a - * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch - * things up to make the test happy. We allow up to 1 jiffy. - * - * This is a regrssion from the timespec->ktime conversion. - */ - if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000) - *timeout = 0; - } - - if (IS_RPS_USER(rps) && - req->fence.seqno == req->engine->last_submitted_seqno) { - /* The GPU is now idle and this client has stalled. - * Since no other client has submitted a request in the - * meantime, assume that this client is the only one - * supplying work to the GPU but is unable to keep that - * work supplied because it is waiting. Since the GPU is - * then never kept fully busy, RPS autoclocking will - * keep the clocks relatively low, causing further delays. - * Compensate by giving the synchronous client credit for - * a waitboost next time. - */ - spin_lock(&req->i915->rps.client_lock); - list_del_init(&rps->link); - spin_unlock(&req->i915->rps.client_lock); - } - - return ret; + return timeout; } -static bool engine_retire_requests(struct intel_engine_cs *engine) +static void engine_retire_requests(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request, *next; - list_for_each_entry_safe(request, next, &engine->request_list, link) { - if (!i915_gem_request_completed(request)) - return false; + list_for_each_entry_safe(request, next, + &engine->timeline->requests, link) { + if (!__i915_gem_request_completed(request)) + return; i915_gem_request_retire(request); } - - return true; } void i915_gem_retire_requests(struct drm_i915_private *dev_priv) { struct intel_engine_cs *engine; - unsigned int tmp; + enum intel_engine_id id; lockdep_assert_held(&dev_priv->drm.struct_mutex); - if (dev_priv->gt.active_engines == 0) + if (!dev_priv->gt.active_requests) return; - GEM_BUG_ON(!dev_priv->gt.awake); - - for_each_engine_masked(engine, dev_priv, dev_priv->gt.active_engines, tmp) - if (engine_retire_requests(engine)) - dev_priv->gt.active_engines &= ~intel_engine_flag(engine); - - if (dev_priv->gt.active_engines == 0) - queue_delayed_work(dev_priv->wq, - &dev_priv->gt.idle_work, - msecs_to_jiffies(100)); + for_each_engine(engine, dev_priv, id) + engine_retire_requests(engine); } diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h index bceeaa3a5193..e2b077df2da0 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.h +++ b/drivers/gpu/drm/i915/i915_gem_request.h @@ -30,6 +30,9 @@ #include "i915_gem.h" #include "i915_sw_fence.h" +struct drm_file; +struct drm_i915_gem_object; + struct intel_wait { struct rb_node node; struct task_struct *tsk; @@ -41,6 +44,33 @@ struct intel_signal_node { struct intel_wait wait; }; +struct i915_dependency { + struct i915_priotree *signaler; + struct list_head signal_link; + struct list_head wait_link; + struct list_head dfs_link; + unsigned long flags; +#define I915_DEPENDENCY_ALLOC BIT(0) +}; + +/* Requests exist in a complex web of interdependencies. Each request + * has to wait for some other request to complete before it is ready to be run + * (e.g. we have to wait until the pixels have been rendering into a texture + * before we can copy from it). We track the readiness of a request in terms + * of fences, but we also need to keep the dependency tree for the lifetime + * of the request (beyond the life of an individual fence). We use the tree + * at various points to reorder the requests whilst keeping the requests + * in order with respect to their various dependencies. + */ +struct i915_priotree { + struct list_head signalers_list; /* those before us, we depend upon */ + struct list_head waiters_list; /* those after us, they depend upon us */ + struct rb_node node; + int priority; +#define I915_PRIORITY_MAX 1024 +#define I915_PRIORITY_MIN (-I915_PRIORITY_MAX) +}; + /** * Request queue structure. * @@ -81,10 +111,39 @@ struct drm_i915_gem_request { struct i915_gem_context *ctx; struct intel_engine_cs *engine; struct intel_ring *ring; + struct intel_timeline *timeline; struct intel_signal_node signaling; + /* Fences for the various phases in the request's lifetime. + * + * The submit fence is used to await upon all of the request's + * dependencies. When it is signaled, the request is ready to run. + * It is used by the driver to then queue the request for execution. + * + * The execute fence is used to signal when the request has been + * sent to hardware. + * + * It is illegal for the submit fence of one request to wait upon the + * execute fence of an earlier request. It should be sufficient to + * wait upon the submit fence of the earlier request. + */ struct i915_sw_fence submit; + struct i915_sw_fence execute; wait_queue_t submitq; + wait_queue_t execq; + + /* A list of everyone we wait upon, and everyone who waits upon us. + * Even though we will not be submitted to the hardware before the + * submit fence is signaled (it waits for all external events as well + * as our own requests), the scheduler still needs to know the + * dependency tree for the lifetime of the request (from execbuf + * to retirement), i.e. bidirectional dependency information for the + * request not tied to individual fences. + */ + struct i915_priotree priotree; + struct i915_dependency dep; + + u32 global_seqno; /** GEM sequence number associated with the previous request, * when the HWS breadcrumb is equal to this the GPU is processing @@ -140,14 +199,11 @@ struct drm_i915_gem_request { struct drm_i915_file_private *file_priv; /** file_priv list entry for this request */ struct list_head client_list; - - /** Link in the execlist submission queue, guarded by execlist_lock. */ - struct list_head execlist_link; }; extern const struct dma_fence_ops i915_fence_ops; -static inline bool fence_is_i915(struct dma_fence *fence) +static inline bool dma_fence_is_i915(const struct dma_fence *fence) { return fence->ops == &i915_fence_ops; } @@ -159,24 +215,12 @@ int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, struct drm_file *file); void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); -static inline u32 -i915_gem_request_get_seqno(struct drm_i915_gem_request *req) -{ - return req ? req->fence.seqno : 0; -} - -static inline struct intel_engine_cs * -i915_gem_request_get_engine(struct drm_i915_gem_request *req) -{ - return req ? req->engine : NULL; -} - static inline struct drm_i915_gem_request * to_request(struct dma_fence *fence) { /* We assume that NULL fence/request are interoperable */ BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0); - GEM_BUG_ON(fence && !fence_is_i915(fence)); + GEM_BUG_ON(fence && !dma_fence_is_i915(fence)); return container_of(fence, struct drm_i915_gem_request, fence); } @@ -214,6 +258,8 @@ int i915_gem_request_await_object(struct drm_i915_gem_request *to, struct drm_i915_gem_object *obj, bool write); +int i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req, + struct dma_fence *fence); void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches); #define i915_add_request(req) \ @@ -221,18 +267,21 @@ void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches); #define i915_add_request_no_flush(req) \ __i915_add_request(req, false) +void __i915_gem_request_submit(struct drm_i915_gem_request *request); +void i915_gem_request_submit(struct drm_i915_gem_request *request); + struct intel_rps_client; #define NO_WAITBOOST ERR_PTR(-1) #define IS_RPS_CLIENT(p) (!IS_ERR(p)) #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p)) -int i915_wait_request(struct drm_i915_gem_request *req, - unsigned int flags, - s64 *timeout, - struct intel_rps_client *rps) +long i915_wait_request(struct drm_i915_gem_request *req, + unsigned int flags, + long timeout) __attribute__((nonnull(1))); #define I915_WAIT_INTERRUPTIBLE BIT(0) #define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */ +#define I915_WAIT_ALL BIT(2) /* used by i915_gem_object_wait() */ static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine); @@ -245,17 +294,37 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2) } static inline bool -i915_gem_request_started(const struct drm_i915_gem_request *req) +__i915_gem_request_started(const struct drm_i915_gem_request *req) { + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), req->previous_seqno); } static inline bool -i915_gem_request_completed(const struct drm_i915_gem_request *req) +i915_gem_request_started(const struct drm_i915_gem_request *req) { + if (!req->global_seqno) + return false; + + return __i915_gem_request_started(req); +} + +static inline bool +__i915_gem_request_completed(const struct drm_i915_gem_request *req) +{ + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), - req->fence.seqno); + req->global_seqno); +} + +static inline bool +i915_gem_request_completed(const struct drm_i915_gem_request *req) +{ + if (!req->global_seqno) + return false; + + return __i915_gem_request_completed(req); } bool __i915_spin_request(const struct drm_i915_gem_request *request, @@ -263,7 +332,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *request, static inline bool i915_spin_request(const struct drm_i915_gem_request *request, int state, unsigned long timeout_us) { - return (i915_gem_request_started(request) && + return (__i915_gem_request_started(request) && __i915_spin_request(request, state, timeout_us)); } @@ -552,53 +621,13 @@ i915_gem_active_isset(const struct i915_gem_active *active) } /** - * i915_gem_active_is_idle - report whether the active tracker is idle - * @active - the active tracker - * - * i915_gem_active_is_idle() returns true if the active tracker is currently - * unassigned or if the request is complete (but not yet retired). Requires - * the caller to hold struct_mutex (but that can be relaxed if desired). - */ -static inline bool -i915_gem_active_is_idle(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return !i915_gem_active_peek(active, mutex); -} - -/** * i915_gem_active_wait - waits until the request is completed * @active - the active request on which to wait - * - * i915_gem_active_wait() waits until the request is completed before - * returning. Note that it does not guarantee that the request is - * retired first, see i915_gem_active_retire(). - * - * i915_gem_active_wait() returns immediately if the active - * request is already complete. - */ -static inline int __must_check -i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex) -{ - struct drm_i915_gem_request *request; - - request = i915_gem_active_peek(active, mutex); - if (!request) - return 0; - - return i915_wait_request(request, - I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, - NULL, NULL); -} - -/** - * i915_gem_active_wait_unlocked - waits until the request is completed - * @active - the active request on which to wait * @flags - how to wait * @timeout - how long to wait at most * @rps - userspace client to charge for a waitboost * - * i915_gem_active_wait_unlocked() waits until the request is completed before + * i915_gem_active_wait() waits until the request is completed before * returning, without requiring any locks to be held. Note that it does not * retire any requests before returning. * @@ -614,21 +643,18 @@ i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex) * Returns 0 if successful, or a negative error code. */ static inline int -i915_gem_active_wait_unlocked(const struct i915_gem_active *active, - unsigned int flags, - s64 *timeout, - struct intel_rps_client *rps) +i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags) { struct drm_i915_gem_request *request; - int ret = 0; + long ret = 0; request = i915_gem_active_get_unlocked(active); if (request) { - ret = i915_wait_request(request, flags, timeout, rps); + ret = i915_wait_request(request, flags, MAX_SCHEDULE_TIMEOUT); i915_gem_request_put(request); } - return ret; + return ret < 0 ? ret : 0; } /** @@ -645,7 +671,7 @@ i915_gem_active_retire(struct i915_gem_active *active, struct mutex *mutex) { struct drm_i915_gem_request *request; - int ret; + long ret; request = i915_gem_active_raw(active, mutex); if (!request) @@ -653,8 +679,8 @@ i915_gem_active_retire(struct i915_gem_active *active, ret = i915_wait_request(request, I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, - NULL, NULL); - if (ret) + MAX_SCHEDULE_TIMEOUT); + if (ret < 0) return ret; list_del_init(&active->link); @@ -665,24 +691,6 @@ i915_gem_active_retire(struct i915_gem_active *active, return 0; } -/* Convenience functions for peeking at state inside active's request whilst - * guarded by the struct_mutex. - */ - -static inline uint32_t -i915_gem_active_get_seqno(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return i915_gem_request_get_seqno(i915_gem_active_peek(active, mutex)); -} - -static inline struct intel_engine_cs * -i915_gem_active_get_engine(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return i915_gem_request_get_engine(i915_gem_active_peek(active, mutex)); -} - #define for_each_active(mask, idx) \ for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index de25b6e0a101..a6fc1bdc48af 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -48,6 +48,20 @@ static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) #endif } +static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) +{ + if (!mutex_trylock(&dev->struct_mutex)) { + if (!mutex_is_locked_by(&dev->struct_mutex, current)) + return false; + + *unlock = false; + } else { + *unlock = true; + } + + return true; +} + static bool any_vma_pinned(struct drm_i915_gem_object *obj) { struct i915_vma *vma; @@ -66,8 +80,11 @@ static bool swap_available(void) static bool can_release_pages(struct drm_i915_gem_object *obj) { - /* Only shmemfs objects are backed by swap */ - if (!obj->base.filp) + if (!obj->mm.pages) + return false; + + /* Consider only shrinkable ojects. */ + if (!i915_gem_object_is_shrinkable(obj)) return false; /* Only report true if by unbinding the object and putting its pages @@ -78,7 +95,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) * to the GPU, simply unbinding from the GPU is not going to succeed * in releasing our pin count on the pages themselves. */ - if (obj->pages_pin_count > obj->bind_count) + if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count) return false; if (any_vma_pinned(obj)) @@ -88,7 +105,14 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) * discard the contents (because the user has marked them as being * purgeable) or if we can move their contents out to swap. */ - return swap_available() || obj->madv == I915_MADV_DONTNEED; + return swap_available() || obj->mm.madv == I915_MADV_DONTNEED; +} + +static bool unsafe_drop_pages(struct drm_i915_gem_object *obj) +{ + if (i915_gem_object_unbind(obj) == 0) + __i915_gem_object_put_pages(obj, I915_MM_SHRINKER); + return !READ_ONCE(obj->mm.pages); } /** @@ -128,6 +152,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, { NULL, 0 }, }, *phase; unsigned long count = 0; + bool unlock; + + if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock)) + return 0; trace_i915_gem_shrink(dev_priv, target, flags); i915_gem_retire_requests(dev_priv); @@ -171,15 +199,19 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, while (count < target && (obj = list_first_entry_or_null(phase->list, typeof(*obj), - global_list))) { - list_move_tail(&obj->global_list, &still_in_list); + global_link))) { + list_move_tail(&obj->global_link, &still_in_list); + if (!obj->mm.pages) { + list_del_init(&obj->global_link); + continue; + } if (flags & I915_SHRINK_PURGEABLE && - obj->madv != I915_MADV_DONTNEED) + obj->mm.madv != I915_MADV_DONTNEED) continue; if (flags & I915_SHRINK_VMAPS && - !is_vmalloc_addr(obj->mapping)) + !is_vmalloc_addr(obj->mm.mapping)) continue; if (!(flags & I915_SHRINK_ACTIVE) && @@ -190,22 +222,28 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, if (!can_release_pages(obj)) continue; - i915_gem_object_get(obj); - - /* For the unbound phase, this should be a no-op! */ - i915_gem_object_unbind(obj); - if (i915_gem_object_put_pages(obj) == 0) - count += obj->base.size >> PAGE_SHIFT; - - i915_gem_object_put(obj); + if (unsafe_drop_pages(obj)) { + /* May arrive from get_pages on another bo */ + mutex_lock_nested(&obj->mm.lock, + I915_MM_SHRINKER); + if (!obj->mm.pages) { + __i915_gem_object_invalidate(obj); + list_del_init(&obj->global_link); + count += obj->base.size >> PAGE_SHIFT; + } + mutex_unlock(&obj->mm.lock); + } } - list_splice(&still_in_list, phase->list); + list_splice_tail(&still_in_list, phase->list); } if (flags & I915_SHRINK_BOUND) intel_runtime_pm_put(dev_priv); i915_gem_retire_requests(dev_priv); + if (unlock) + mutex_unlock(&dev_priv->drm.struct_mutex); + /* expedite the RCU grace period to free some request slabs */ synchronize_rcu_expedited(); @@ -239,19 +277,6 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) return freed; } -static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) -{ - if (!mutex_trylock(&dev->struct_mutex)) { - if (!mutex_is_locked_by(&dev->struct_mutex, current)) - return false; - - *unlock = false; - } else - *unlock = true; - - return true; -} - static unsigned long i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { @@ -268,11 +293,11 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) i915_gem_retire_requests(dev_priv); count = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) if (can_release_pages(obj)) count += obj->base.size >> PAGE_SHIFT; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) count += obj->base.size >> PAGE_SHIFT; } @@ -373,13 +398,19 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) * being pointed to by hardware. */ unbound = bound = unevictable = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { + if (!obj->mm.pages) + continue; + if (!can_release_pages(obj)) unevictable += obj->base.size >> PAGE_SHIFT; else unbound += obj->base.size >> PAGE_SHIFT; } - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { + if (!obj->mm.pages) + continue; + if (!can_release_pages(obj)) unevictable += obj->base.size >> PAGE_SHIFT; else diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index f4f6d3a48b05..ebaa941c83af 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -89,9 +89,8 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, mutex_unlock(&dev_priv->mm.stolen_lock); } -static unsigned long i915_stolen_to_physical(struct drm_device *dev) +static unsigned long i915_stolen_to_physical(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = dev_priv->drm.pdev; struct i915_ggtt *ggtt = &dev_priv->ggtt; struct resource *r; @@ -109,7 +108,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) * */ base = 0; - if (INTEL_INFO(dev)->gen >= 3) { + if (INTEL_GEN(dev_priv) >= 3) { u32 bsm; pci_read_config_dword(pdev, INTEL_BSM, &bsm); @@ -138,7 +137,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) I865_TOUD, &toud); base = (toud << 16) + tseg_size; - } else if (IS_I85X(dev)) { + } else if (IS_I85X(dev_priv)) { u32 tseg_size = 0; u32 tom; u8 tmp; @@ -253,7 +252,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) * kernel. So if the region is already marked as busy, something * is seriously wrong. */ - r = devm_request_mem_region(dev->dev, base, ggtt->stolen_size, + r = devm_request_mem_region(dev_priv->drm.dev, base, ggtt->stolen_size, "Graphics Stolen Memory"); if (r == NULL) { /* @@ -264,7 +263,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) * PCI bus, but have an off-by-one error. Hence retry the * reservation starting from 1 instead of 0. */ - r = devm_request_mem_region(dev->dev, base + 1, + r = devm_request_mem_region(dev_priv->drm.dev, base + 1, ggtt->stolen_size - 1, "Graphics Stolen Memory"); /* @@ -408,9 +407,8 @@ static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv, *size = stolen_top - *base; } -int i915_gem_init_stolen(struct drm_device *dev) +int i915_gem_init_stolen(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; unsigned long reserved_total, reserved_base = 0, reserved_size; unsigned long stolen_top; @@ -418,7 +416,7 @@ int i915_gem_init_stolen(struct drm_device *dev) mutex_init(&dev_priv->mm.stolen_lock); #ifdef CONFIG_INTEL_IOMMU - if (intel_iommu_gfx_mapped && INTEL_INFO(dev)->gen < 8) { + if (intel_iommu_gfx_mapped && INTEL_GEN(dev_priv) < 8) { DRM_INFO("DMAR active, disabling use of stolen memory\n"); return 0; } @@ -427,7 +425,7 @@ int i915_gem_init_stolen(struct drm_device *dev) if (ggtt->stolen_size == 0) return 0; - dev_priv->mm.stolen_base = i915_stolen_to_physical(dev); + dev_priv->mm.stolen_base = i915_stolen_to_physical(dev_priv); if (dev_priv->mm.stolen_base == 0) return 0; @@ -515,12 +513,10 @@ i915_pages_create_for_stolen(struct drm_device *dev, u32 offset, u32 size) { struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; struct sg_table *st; struct scatterlist *sg; - DRM_DEBUG_DRIVER("offset=0x%x, size=%d\n", offset, size); - BUG_ON(offset > ggtt->stolen_size - size); + GEM_BUG_ON(offset > dev_priv->ggtt.stolen_size - size); /* We hide that we have no struct page backing our stolen object * by wrapping the contiguous physical allocation with a fake @@ -529,11 +525,11 @@ i915_pages_create_for_stolen(struct drm_device *dev, st = kmalloc(sizeof(*st), GFP_KERNEL); if (st == NULL) - return NULL; + return ERR_PTR(-ENOMEM); if (sg_alloc_table(st, 1, GFP_KERNEL)) { kfree(st); - return NULL; + return ERR_PTR(-ENOMEM); } sg = st->sgl; @@ -546,31 +542,36 @@ i915_pages_create_for_stolen(struct drm_device *dev, return st; } -static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) { - BUG(); - return -EINVAL; + return i915_pages_create_for_stolen(obj->base.dev, + obj->stolen->start, + obj->stolen->size); } -static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - /* Should only be called during free */ - sg_free_table(obj->pages); - kfree(obj->pages); + /* Should only be called from i915_gem_object_release_stolen() */ + sg_free_table(pages); + kfree(pages); } - static void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); + struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen); - if (obj->stolen) { - i915_gem_stolen_remove_node(dev_priv, obj->stolen); - kfree(obj->stolen); - obj->stolen = NULL; - } + GEM_BUG_ON(!stolen); + + __i915_gem_object_unpin_pages(obj); + + i915_gem_stolen_remove_node(dev_priv, stolen); + kfree(stolen); } + static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = { .get_pages = i915_gem_object_get_pages_stolen, .put_pages = i915_gem_object_put_pages_stolen, @@ -590,19 +591,13 @@ _i915_gem_object_create_stolen(struct drm_device *dev, drm_gem_private_object_init(dev, &obj->base, stolen->size); i915_gem_object_init(obj, &i915_gem_object_stolen_ops); - obj->pages = i915_pages_create_for_stolen(dev, - stolen->start, stolen->size); - if (obj->pages == NULL) - goto cleanup; - - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - - i915_gem_object_pin_pages(obj); obj->stolen = stolen; - obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; - obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE; + obj->cache_level = HAS_LLC(to_i915(dev)) ? + I915_CACHE_LLC : I915_CACHE_NONE; + + if (i915_gem_object_pin_pages(obj)) + goto cleanup; return obj; @@ -622,7 +617,6 @@ i915_gem_object_create_stolen(struct drm_device *dev, u32 size) if (!drm_mm_initialized(&dev_priv->mm.stolen)) return NULL; - DRM_DEBUG_KMS("creating stolen object: size=%x\n", size); if (size == 0) return NULL; @@ -698,10 +692,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, if (gtt_offset == I915_GTT_OFFSET_NONE) return obj; + ret = i915_gem_object_pin_pages(obj); + if (ret) + goto err; + vma = i915_gem_obj_lookup_or_create_vma(obj, &ggtt->base, NULL); if (IS_ERR(vma)) { ret = PTR_ERR(vma); - goto err; + goto err_pages; } /* To simplify the initialisation sequence between KMS and GTT, @@ -715,20 +713,20 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, ret = drm_mm_reserve_node(&ggtt->base.mm, &vma->node); if (ret) { DRM_DEBUG_KMS("failed to allocate stolen GTT space\n"); - goto err; + goto err_pages; } - vma->pages = obj->pages; + vma->pages = obj->mm.pages; vma->flags |= I915_VMA_GLOBAL_BIND; __i915_vma_set_map_and_fenceable(vma); list_move_tail(&vma->vm_link, &ggtt->base.inactive_list); + list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); obj->bind_count++; - list_add_tail(&obj->global_list, &dev_priv->mm.bound_list); - i915_gem_object_pin_pages(obj); - return obj; +err_pages: + i915_gem_object_unpin_pages(obj); err: i915_gem_object_put(obj); return NULL; diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index c21bc0068d20..c85e7b06bdba 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -60,9 +60,9 @@ /* Check pitch constriants for all chips & tiling formats */ static bool -i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) +i915_tiling_ok(struct drm_i915_private *dev_priv, + int stride, int size, int tiling_mode) { - struct drm_i915_private *dev_priv = to_i915(dev); int tile_width; /* Linear is always fine */ @@ -81,10 +81,10 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) /* check maximum stride & object size */ /* i965+ stores the end address of the gtt mapping in the fence * reg, so dont bother to check the size */ - if (INTEL_INFO(dev)->gen >= 7) { + if (INTEL_GEN(dev_priv) >= 7) { if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL) return false; - } else if (INTEL_INFO(dev)->gen >= 4) { + } else if (INTEL_GEN(dev_priv) >= 4) { if (stride / 128 > I965_FENCE_MAX_PITCH_VAL) return false; } else { @@ -104,7 +104,7 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) return false; /* 965+ just needs multiples of tile width */ - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { if (stride & (tile_width - 1)) return false; return true; @@ -199,14 +199,12 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, if (!obj) return -ENOENT; - if (!i915_tiling_ok(dev, + if (!i915_tiling_ok(dev_priv, args->stride, obj->base.size, args->tiling_mode)) { - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return -EINVAL; } - intel_runtime_pm_get(dev_priv); - mutex_lock(&dev->struct_mutex); if (obj->pin_display || obj->framebuffer_references) { err = -EBUSY; @@ -261,14 +259,22 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, if (!err) { struct i915_vma *vma; - if (obj->pages && - obj->madv == I915_MADV_WILLNEED && + mutex_lock(&obj->mm.lock); + if (obj->mm.pages && + obj->mm.madv == I915_MADV_WILLNEED && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { - if (args->tiling_mode == I915_TILING_NONE) - i915_gem_object_unpin_pages(obj); - if (!i915_gem_object_is_tiled(obj)) - i915_gem_object_pin_pages(obj); + if (args->tiling_mode == I915_TILING_NONE) { + GEM_BUG_ON(!obj->mm.quirked); + __i915_gem_object_unpin_pages(obj); + obj->mm.quirked = false; + } + if (!i915_gem_object_is_tiled(obj)) { + GEM_BUG_ON(!obj->mm.quirked); + __i915_gem_object_pin_pages(obj); + obj->mm.quirked = true; + } } + mutex_unlock(&obj->mm.lock); list_for_each_entry(vma, &obj->vma_list, obj_link) { if (!vma->fence) @@ -302,8 +308,6 @@ err: i915_gem_object_put(obj); mutex_unlock(&dev->struct_mutex); - intel_runtime_pm_put(dev_priv); - return err; } @@ -327,12 +331,19 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_i915_gem_get_tiling *args = data; struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; + int err = -ENOENT; + + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); + if (obj) { + args->tiling_mode = + READ_ONCE(obj->tiling_and_stride) & TILING_MASK; + err = 0; + } + rcu_read_unlock(); + if (unlikely(err)) + return err; - obj = i915_gem_object_lookup(file, args->handle); - if (!obj) - return -ENOENT; - - args->tiling_mode = READ_ONCE(obj->tiling_and_stride) & TILING_MASK; switch (args->tiling_mode) { case I915_TILING_X: args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; @@ -340,11 +351,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, case I915_TILING_Y: args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; break; + default: case I915_TILING_NONE: args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; break; - default: - DRM_ERROR("unknown tiling mode\n"); } /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ @@ -357,6 +367,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; - i915_gem_object_put_unlocked(obj); return 0; } diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c new file mode 100644 index 000000000000..bf8a471b61e6 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_timeline.c @@ -0,0 +1,90 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "i915_drv.h" + +static int __i915_gem_timeline_init(struct drm_i915_private *i915, + struct i915_gem_timeline *timeline, + const char *name, + struct lock_class_key *lockclass, + const char *lockname) +{ + unsigned int i; + u64 fences; + + lockdep_assert_held(&i915->drm.struct_mutex); + + timeline->i915 = i915; + timeline->name = kstrdup(name ?: "[kernel]", GFP_KERNEL); + if (!timeline->name) + return -ENOMEM; + + list_add(&timeline->link, &i915->gt.timelines); + + /* Called during early_init before we know how many engines there are */ + fences = dma_fence_context_alloc(ARRAY_SIZE(timeline->engine)); + for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) { + struct intel_timeline *tl = &timeline->engine[i]; + + tl->fence_context = fences++; + tl->common = timeline; +#ifdef CONFIG_DEBUG_SPINLOCK + __raw_spin_lock_init(&tl->lock.rlock, lockname, lockclass); +#else + spin_lock_init(&tl->lock); +#endif + init_request_active(&tl->last_request, NULL); + INIT_LIST_HEAD(&tl->requests); + } + + return 0; +} + +int i915_gem_timeline_init(struct drm_i915_private *i915, + struct i915_gem_timeline *timeline, + const char *name) +{ + static struct lock_class_key class; + + return __i915_gem_timeline_init(i915, timeline, name, + &class, "&timeline->lock"); +} + +int i915_gem_timeline_init__global(struct drm_i915_private *i915) +{ + static struct lock_class_key class; + + return __i915_gem_timeline_init(i915, + &i915->gt.global_timeline, + "[execution]", + &class, "&global_timeline->lock"); +} + +void i915_gem_timeline_fini(struct i915_gem_timeline *tl) +{ + lockdep_assert_held(&tl->i915->drm.struct_mutex); + + list_del(&tl->link); + kfree(tl->name); +} diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h new file mode 100644 index 000000000000..98d99a62b4ae --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_timeline.h @@ -0,0 +1,73 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#ifndef I915_GEM_TIMELINE_H +#define I915_GEM_TIMELINE_H + +#include <linux/list.h> + +#include "i915_gem_request.h" + +struct i915_gem_timeline; + +struct intel_timeline { + u64 fence_context; + u32 last_submitted_seqno; + + spinlock_t lock; + + /** + * List of breadcrumbs associated with GPU requests currently + * outstanding. + */ + struct list_head requests; + + /* Contains an RCU guarded pointer to the last request. No reference is + * held to the request, users must carefully acquire a reference to + * the request using i915_gem_active_get_request_rcu(), or hold the + * struct_mutex. + */ + struct i915_gem_active last_request; + u32 sync_seqno[I915_NUM_ENGINES]; + + struct i915_gem_timeline *common; +}; + +struct i915_gem_timeline { + struct list_head link; + atomic_t next_seqno; + + struct drm_i915_private *i915; + const char *name; + + struct intel_timeline engine[I915_NUM_ENGINES]; +}; + +int i915_gem_timeline_init(struct drm_i915_private *i915, + struct i915_gem_timeline *tl, + const char *name); +int i915_gem_timeline_init__global(struct drm_i915_private *i915); +void i915_gem_timeline_fini(struct i915_gem_timeline *tl); + +#endif diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index c6f780f5abc9..107ddf51065e 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -61,33 +61,26 @@ struct i915_mmu_object { bool attached; }; -static void wait_rendering(struct drm_i915_gem_object *obj) -{ - unsigned long active = __I915_BO_ACTIVE(obj); - int idx; - - for_each_active(active, idx) - i915_gem_active_wait_unlocked(&obj->last_read[idx], - 0, NULL, NULL); -} - static void cancel_userptr(struct work_struct *work) { struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); struct drm_i915_gem_object *obj = mo->obj; struct drm_device *dev = obj->base.dev; - wait_rendering(obj); + i915_gem_object_wait(obj, I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT, NULL); mutex_lock(&dev->struct_mutex); /* Cancel any active worker and force us to re-evaluate gup */ obj->userptr.work = NULL; - if (obj->pages != NULL) { - /* We are inside a kthread context and can't be interrupted */ - WARN_ON(i915_gem_object_unbind(obj)); - WARN_ON(i915_gem_object_put_pages(obj)); - } + /* We are inside a kthread context and can't be interrupted */ + if (i915_gem_object_unbind(obj) == 0) + __i915_gem_object_put_pages(obj, I915_MM_NORMAL); + WARN_ONCE(obj->mm.pages, + "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n", + obj->bind_count, + atomic_read(&obj->mm.pages_pin_count), + obj->pin_display); i915_gem_object_put(obj); mutex_unlock(&dev->struct_mutex); @@ -436,24 +429,25 @@ err: return ret; } -static int +static struct sg_table * __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, struct page **pvec, int num_pages) { + struct sg_table *pages; int ret; - ret = st_set_pages(&obj->pages, pvec, num_pages); + ret = st_set_pages(&pages, pvec, num_pages); if (ret) - return ret; + return ERR_PTR(ret); - ret = i915_gem_gtt_prepare_object(obj); + ret = i915_gem_gtt_prepare_pages(obj, pages); if (ret) { - sg_free_table(obj->pages); - kfree(obj->pages); - obj->pages = NULL; + sg_free_table(pages); + kfree(pages); + return ERR_PTR(ret); } - return ret; + return pages; } static int @@ -497,7 +491,6 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) { struct get_pages_work *work = container_of(_work, typeof(*work), work); struct drm_i915_gem_object *obj = work->obj; - struct drm_device *dev = obj->base.dev; const int npages = obj->base.size >> PAGE_SHIFT; struct page **pvec; int pinned, ret; @@ -533,33 +526,32 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) } } - mutex_lock(&dev->struct_mutex); + mutex_lock(&obj->mm.lock); if (obj->userptr.work == &work->work) { + struct sg_table *pages = ERR_PTR(ret); + if (pinned == npages) { - ret = __i915_gem_userptr_set_pages(obj, pvec, npages); - if (ret == 0) { - list_add_tail(&obj->global_list, - &to_i915(dev)->mm.unbound_list); - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; + pages = __i915_gem_userptr_set_pages(obj, pvec, npages); + if (!IS_ERR(pages)) { + __i915_gem_object_set_pages(obj, pages); pinned = 0; + pages = NULL; } } - obj->userptr.work = ERR_PTR(ret); - } - obj->userptr.workers--; - i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); + obj->userptr.work = ERR_CAST(pages); + } + mutex_unlock(&obj->mm.lock); release_pages(pvec, pinned, 0); drm_free_large(pvec); + i915_gem_object_put(obj); put_task_struct(work->task); kfree(work); } -static int +static struct sg_table * __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, bool *active) { @@ -584,15 +576,11 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, * that error back to this function through * obj->userptr.work = ERR_PTR. */ - if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS) - return -EAGAIN; - work = kmalloc(sizeof(*work), GFP_KERNEL); if (work == NULL) - return -ENOMEM; + return ERR_PTR(-ENOMEM); obj->userptr.work = &work->work; - obj->userptr.workers++; work->obj = i915_gem_object_get(obj); @@ -603,14 +591,15 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, schedule_work(&work->work); *active = true; - return -EAGAIN; + return ERR_PTR(-EAGAIN); } -static int +static struct sg_table * i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) { const int num_pages = obj->base.size >> PAGE_SHIFT; struct page **pvec; + struct sg_table *pages; int pinned, ret; bool active; @@ -634,15 +623,15 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) if (obj->userptr.work) { /* active flag should still be held for the pending work */ if (IS_ERR(obj->userptr.work)) - return PTR_ERR(obj->userptr.work); + return ERR_CAST(obj->userptr.work); else - return -EAGAIN; + return ERR_PTR(-EAGAIN); } /* Let the mmu-notifier know that we have begun and need cancellation */ ret = __i915_gem_userptr_set_active(obj, true); if (ret) - return ret; + return ERR_PTR(ret); pvec = NULL; pinned = 0; @@ -651,7 +640,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) GFP_TEMPORARY); if (pvec == NULL) { __i915_gem_userptr_set_active(obj, false); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, @@ -660,21 +649,22 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) active = false; if (pinned < 0) - ret = pinned, pinned = 0; + pages = ERR_PTR(pinned), pinned = 0; else if (pinned < num_pages) - ret = __i915_gem_userptr_get_pages_schedule(obj, &active); + pages = __i915_gem_userptr_get_pages_schedule(obj, &active); else - ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); - if (ret) { + pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages); + if (IS_ERR(pages)) { __i915_gem_userptr_set_active(obj, active); release_pages(pvec, pinned, 0); } drm_free_large(pvec); - return ret; + return pages; } static void -i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) +i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct sgt_iter sgt_iter; struct page *page; @@ -682,22 +672,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) BUG_ON(obj->userptr.work != NULL); __i915_gem_userptr_set_active(obj, false); - if (obj->madv != I915_MADV_WILLNEED) - obj->dirty = 0; + if (obj->mm.madv != I915_MADV_WILLNEED) + obj->mm.dirty = false; - i915_gem_gtt_finish_object(obj); + i915_gem_gtt_finish_pages(obj, pages); - for_each_sgt_page(page, sgt_iter, obj->pages) { - if (obj->dirty) + for_each_sgt_page(page, sgt_iter, pages) { + if (obj->mm.dirty) set_page_dirty(page); mark_page_accessed(page); put_page(page); } - obj->dirty = 0; + obj->mm.dirty = false; - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(pages); + kfree(pages); } static void @@ -717,7 +707,8 @@ i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) } static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { - .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, + .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = i915_gem_userptr_get_pages, .put_pages = i915_gem_userptr_put_pages, .dmabuf_export = i915_gem_userptr_dmabuf_export, @@ -762,12 +753,13 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_userptr *args = data; struct drm_i915_gem_object *obj; int ret; u32 handle; - if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) { + if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) { /* We cannot support coherent userptr objects on hw without * LLC and broken snooping. */ @@ -816,7 +808,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file ret = drm_gem_handle_create(file, &obj->base, &handle); /* drop reference from allocate - handle holds it now */ - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 242b9a927899..ae84aa4b1467 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -415,17 +415,13 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, if (INTEL_GEN(m->i915) >= 6) { err_printf(m, " RC PSMI: 0x%08x\n", ee->rc_psmi); err_printf(m, " FAULT_REG: 0x%08x\n", ee->fault_reg); - err_printf(m, " SYNC_0: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[0], - ee->semaphore_seqno[0]); - err_printf(m, " SYNC_1: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[1], - ee->semaphore_seqno[1]); - if (HAS_VEBOX(m->i915)) { - err_printf(m, " SYNC_2: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[2], - ee->semaphore_seqno[2]); - } + err_printf(m, " SYNC_0: 0x%08x\n", + ee->semaphore_mboxes[0]); + err_printf(m, " SYNC_1: 0x%08x\n", + ee->semaphore_mboxes[1]); + if (HAS_VEBOX(m->i915)) + err_printf(m, " SYNC_2: 0x%08x\n", + ee->semaphore_mboxes[2]); } if (USES_PPGTT(m->i915)) { err_printf(m, " GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode); @@ -532,8 +528,7 @@ static void err_print_capabilities(struct drm_i915_error_state_buf *m, int i915_error_state_to_str(struct drm_i915_error_state_buf *m, const struct i915_error_state_file_priv *error_priv) { - struct drm_device *dev = error_priv->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(error_priv->dev); struct pci_dev *pdev = dev_priv->drm.pdev; struct drm_i915_error_state *error = error_priv->error; struct drm_i915_error_object *obj; @@ -546,9 +541,13 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, } err_printf(m, "%s\n", error->error_msg); - err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec, - error->time.tv_usec); err_printf(m, "Kernel: " UTS_RELEASE "\n"); + err_printf(m, "Time: %ld s %ld us\n", + error->time.tv_sec, error->time.tv_usec); + err_printf(m, "Boottime: %ld s %ld us\n", + error->boottime.tv_sec, error->boottime.tv_usec); + err_printf(m, "Uptime: %ld s %ld us\n", + error->uptime.tv_sec, error->uptime.tv_usec); err_print_capabilities(m, &error->device_info); max_hangcheck_score = 0; for (i = 0; i < ARRAY_SIZE(error->engine); i++) { @@ -573,7 +572,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, pdev->subsystem_device); err_printf(m, "IOMMU enabled?: %d\n", error->iommu); - if (HAS_CSR(dev)) { + if (HAS_CSR(dev_priv)) { struct intel_csr *csr = &dev_priv->csr; err_printf(m, "DMC loaded: %s\n", @@ -585,7 +584,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, "EIR: 0x%08x\n", error->eir); err_printf(m, "IER: 0x%08x\n", error->ier); - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { for (i = 0; i < 4; i++) err_printf(m, "GTIER gt %d: 0x%08x\n", i, error->gtier[i]); @@ -600,10 +599,10 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, for (i = 0; i < dev_priv->num_fence_regs; i++) err_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]); - if (INTEL_INFO(dev)->gen >= 6) { + if (INTEL_GEN(dev_priv) >= 6) { err_printf(m, "ERROR: 0x%08x\n", error->error); - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_GEN(dev_priv) >= 8) err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n", error->fault_data1, error->fault_data0); @@ -702,11 +701,13 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, print_error_obj(m, NULL, "Semaphores", error->semaphore); + print_error_obj(m, NULL, "GuC log buffer", error->guc_log); + if (error->overlay) intel_overlay_print_error_state(m, error->overlay); if (error->display) - intel_display_print_error_state(m, dev, error->display); + intel_display_print_error_state(m, dev_priv, error->display); out: if (m->bytes == 0 && m->err) @@ -782,6 +783,7 @@ static void i915_error_state_free(struct kref *error_ref) } i915_error_object_free(error->semaphore); + i915_error_object_free(error->guc_log); for (i = 0; i < ARRAY_SIZE(error->active_bo); i++) kfree(error->active_bo[i]); @@ -858,16 +860,19 @@ out: static inline uint32_t __active_get_seqno(struct i915_gem_active *active) { - return i915_gem_request_get_seqno(__i915_gem_active_peek(active)); + struct drm_i915_gem_request *request; + + request = __i915_gem_active_peek(active); + return request ? request->global_seqno : 0; } static inline int __active_get_engine_id(struct i915_gem_active *active) { - struct intel_engine_cs *engine; + struct drm_i915_gem_request *request; - engine = i915_gem_request_get_engine(__i915_gem_active_peek(active)); - return engine ? engine->id : -1; + request = __i915_gem_active_peek(active); + return request ? request->engine->id : -1; } static void capture_bo(struct drm_i915_error_buffer *err, @@ -880,17 +885,17 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->name = obj->base.name; for (i = 0; i < I915_NUM_ENGINES; i++) - err->rseqno[i] = __active_get_seqno(&obj->last_read[i]); - err->wseqno = __active_get_seqno(&obj->last_write); - err->engine = __active_get_engine_id(&obj->last_write); + err->rseqno[i] = __active_get_seqno(&vma->last_read[i]); + err->wseqno = __active_get_seqno(&obj->frontbuffer_write); + err->engine = __active_get_engine_id(&obj->frontbuffer_write); err->gtt_offset = vma->node.start; err->read_domains = obj->base.read_domains; err->write_domain = obj->base.write_domain; err->fence_reg = vma->fence ? vma->fence->id : -1; err->tiling = i915_gem_object_get_tiling(obj); - err->dirty = obj->dirty; - err->purgeable = obj->madv != I915_MADV_WILLNEED; + err->dirty = obj->mm.dirty; + err->purgeable = obj->mm.madv != I915_MADV_WILLNEED; err->userptr = obj->userptr.mm != NULL; err->cache_level = obj->cache_level; } @@ -965,6 +970,26 @@ static void i915_gem_record_fences(struct drm_i915_private *dev_priv, } } +static inline u32 +gen8_engine_sync_index(struct intel_engine_cs *engine, + struct intel_engine_cs *other) +{ + int idx; + + /* + * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2; + * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs; + * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs; + * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs; + * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs; + */ + + idx = (other - engine) - 1; + if (idx < 0) + idx += I915_NUM_ENGINES; + + return idx; +} static void gen8_record_semaphore_state(struct drm_i915_error_state *error, struct intel_engine_cs *engine, @@ -988,10 +1013,9 @@ static void gen8_record_semaphore_state(struct drm_i915_error_state *error, signal_offset = (GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4; tmp = error->semaphore->pages[0]; - idx = intel_engine_sync_index(engine, to); + idx = gen8_engine_sync_index(engine, to); ee->semaphore_mboxes[idx] = tmp[signal_offset]; - ee->semaphore_seqno[idx] = engine->semaphore.sync_seqno[idx]; } } @@ -1002,14 +1026,9 @@ static void gen6_record_semaphore_state(struct intel_engine_cs *engine, ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base)); ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base)); - ee->semaphore_seqno[0] = engine->semaphore.sync_seqno[0]; - ee->semaphore_seqno[1] = engine->semaphore.sync_seqno[1]; - - if (HAS_VEBOX(dev_priv)) { + if (HAS_VEBOX(dev_priv)) ee->semaphore_mboxes[2] = I915_READ(RING_SYNC_2(engine->mmio_base)); - ee->semaphore_seqno[2] = engine->semaphore.sync_seqno[2]; - } } static void error_record_engine_waiters(struct intel_engine_cs *engine, @@ -1026,7 +1045,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (RB_EMPTY_ROOT(&b->waiters)) return; - if (!spin_trylock(&b->lock)) { + if (!spin_trylock_irq(&b->lock)) { ee->waiters = ERR_PTR(-EDEADLK); return; } @@ -1034,7 +1053,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, count = 0; for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb)) count++; - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); waiter = NULL; if (count) @@ -1044,7 +1063,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (!waiter) return; - if (!spin_trylock(&b->lock)) { + if (!spin_trylock_irq(&b->lock)) { kfree(waiter); ee->waiters = ERR_PTR(-EDEADLK); return; @@ -1062,7 +1081,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (++ee->num_waiters == count) break; } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static void error_record_engine_registers(struct drm_i915_error_state *error, @@ -1103,7 +1122,7 @@ static void error_record_engine_registers(struct drm_i915_error_state *error, ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base)); ee->acthd = intel_engine_get_active_head(engine); ee->seqno = intel_engine_get_seqno(engine); - ee->last_seqno = engine->last_submitted_seqno; + ee->last_seqno = intel_engine_last_submit(engine); ee->start = I915_READ_START(engine); ee->head = I915_READ_HEAD(engine); ee->tail = I915_READ_TAIL(engine); @@ -1169,7 +1188,7 @@ static void record_request(struct drm_i915_gem_request *request, struct drm_i915_error_request *erq) { erq->context = request->ctx->hw_id; - erq->seqno = request->fence.seqno; + erq->seqno = request->global_seqno; erq->jiffies = request->emitted_jiffies; erq->head = request->head; erq->tail = request->tail; @@ -1188,7 +1207,7 @@ static void engine_record_requests(struct intel_engine_cs *engine, count = 0; request = first; - list_for_each_entry_from(request, &engine->request_list, link) + list_for_each_entry_from(request, &engine->timeline->requests, link) count++; if (!count) return; @@ -1201,7 +1220,7 @@ static void engine_record_requests(struct intel_engine_cs *engine, count = 0; request = first; - list_for_each_entry_from(request, &engine->request_list, link) { + list_for_each_entry_from(request, &engine->timeline->requests, link) { if (count >= ee->num_requests) { /* * If the ring request list was changed in @@ -1408,11 +1427,21 @@ static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv, error->pinned_bo = bo; } +static void i915_gem_capture_guc_log_buffer(struct drm_i915_private *dev_priv, + struct drm_i915_error_state *error) +{ + /* Capturing log buf contents won't be useful if logging was disabled */ + if (!dev_priv->guc.log.vma || (i915.guc_log_level < 0)) + return; + + error->guc_log = i915_error_object_create(dev_priv, + dev_priv->guc.log.vma); +} + /* Capture all registers which don't fit into another category. */ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error) { - struct drm_device *dev = &dev_priv->drm; int i; /* General organization @@ -1433,7 +1462,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, if (IS_GEN7(dev_priv)) error->err_int = I915_READ(GEN7_ERR_INT); - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0); error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1); } @@ -1445,10 +1474,10 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, } /* 2: Registers which belong to multiple generations */ - if (INTEL_INFO(dev)->gen >= 7) + if (INTEL_GEN(dev_priv) >= 7) error->forcewake = I915_READ_FW(FORCEWAKE_MT); - if (INTEL_INFO(dev)->gen >= 6) { + if (INTEL_GEN(dev_priv) >= 6) { error->derrmr = I915_READ(DERRMR); error->error = I915_READ(ERROR_GEN6); error->done_reg = I915_READ(DONE_REG); @@ -1461,10 +1490,10 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, } /* 4: Everything else */ - if (HAS_HW_CONTEXTS(dev)) + if (HAS_HW_CONTEXTS(dev_priv)) error->ccid = I915_READ(CCID); - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { error->ier = I915_READ(GEN8_DE_MISC_IER); for (i = 0; i < 4; i++) error->gtier[i] = I915_READ(GEN8_GT_IER(i)); @@ -1532,8 +1561,13 @@ static int capture(void *data) i915_gem_record_rings(error->i915, error); i915_capture_active_buffers(error->i915, error); i915_capture_pinned_buffers(error->i915, error); + i915_gem_capture_guc_log_buffer(error->i915, error); do_gettimeofday(&error->time); + error->boottime = ktime_to_timeval(ktime_get_boottime()); + error->uptime = + ktime_to_timeval(ktime_sub(ktime_get(), + error->i915->gt.last_init_time)); error->overlay = intel_overlay_capture_error_state(error->i915); error->display = intel_display_capture_error_state(error->i915); diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index a1f76c8f8cde..4462112725ef 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -23,6 +23,8 @@ */ #include <linux/firmware.h> #include <linux/circ_buf.h> +#include <linux/debugfs.h> +#include <linux/relay.h> #include "i915_drv.h" #include "intel_guc.h" @@ -85,6 +87,7 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len) if (WARN_ON(len < 1 || len > 15)) return -EINVAL; + mutex_lock(&guc->action_lock); intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); dev_priv->guc.action_count += 1; @@ -123,6 +126,7 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len) dev_priv->guc.action_status = status; intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); + mutex_unlock(&guc->action_lock); return ret; } @@ -170,6 +174,35 @@ static int host2guc_sample_forcewake(struct intel_guc *guc, return host2guc_action(guc, data, ARRAY_SIZE(data)); } +static int host2guc_logbuffer_flush_complete(struct intel_guc *guc) +{ + u32 data[1]; + + data[0] = HOST2GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE; + + return host2guc_action(guc, data, 1); +} + +static int host2guc_force_logbuffer_flush(struct intel_guc *guc) +{ + u32 data[2]; + + data[0] = HOST2GUC_ACTION_FORCE_LOG_BUFFER_FLUSH; + data[1] = 0; + + return host2guc_action(guc, data, 2); +} + +static int host2guc_logging_control(struct intel_guc *guc, u32 control_val) +{ + u32 data[2]; + + data[0] = HOST2GUC_ACTION_UK_LOG_ENABLE_LOGGING; + data[1] = control_val; + + return host2guc_action(guc, data, 2); +} + /* * Initialise, update, or clear doorbell data shared with the GuC * @@ -187,7 +220,7 @@ static int guc_update_doorbell_id(struct intel_guc *guc, struct guc_context_desc desc; size_t len; - doorbell = client->client_base + client->doorbell_offset; + doorbell = client->vaddr + client->doorbell_offset; if (client->doorbell_id != GUC_INVALID_DOORBELL_ID && test_bit(client->doorbell_id, doorbell_bitmap)) { @@ -293,7 +326,7 @@ static void guc_proc_desc_init(struct intel_guc *guc, { struct guc_process_desc *desc; - desc = client->client_base + client->proc_desc_offset; + desc = client->vaddr + client->proc_desc_offset; memset(desc, 0, sizeof(*desc)); @@ -380,8 +413,8 @@ static void guc_ctx_desc_init(struct intel_guc *guc, gfx_addr = i915_ggtt_offset(client->vma); desc.db_trigger_phy = sg_dma_address(client->vma->pages->sgl) + client->doorbell_offset; - desc.db_trigger_cpu = (uintptr_t)client->client_base + - client->doorbell_offset; + desc.db_trigger_cpu = + (uintptr_t)client->vaddr + client->doorbell_offset; desc.db_trigger_uk = gfx_addr + client->doorbell_offset; desc.process_desc = gfx_addr + client->proc_desc_offset; desc.wq_addr = gfx_addr + client->wq_offset; @@ -432,7 +465,7 @@ int i915_guc_wq_reserve(struct drm_i915_gem_request *request) { const size_t wqi_size = sizeof(struct guc_wq_item); struct i915_guc_client *gc = request->i915->guc.execbuf_client; - struct guc_process_desc *desc = gc->client_base + gc->proc_desc_offset; + struct guc_process_desc *desc = gc->vaddr + gc->proc_desc_offset; u32 freespace; int ret; @@ -473,10 +506,9 @@ static void guc_wq_item_append(struct i915_guc_client *gc, struct intel_engine_cs *engine = rq->engine; struct guc_process_desc *desc; struct guc_wq_item *wqi; - void *base; - u32 freespace, tail, wq_off, wq_page; + u32 freespace, tail, wq_off; - desc = gc->client_base + gc->proc_desc_offset; + desc = gc->vaddr + gc->proc_desc_offset; /* Free space is guaranteed, see i915_guc_wq_reserve() above */ freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size); @@ -506,10 +538,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc, gc->wq_rsvd -= wqi_size; /* WQ starts from the page after doorbell / process_desc */ - wq_page = (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT; - wq_off &= PAGE_SIZE - 1; - base = kmap_atomic(i915_gem_object_get_page(gc->vma->obj, wq_page)); - wqi = (struct guc_wq_item *)((char *)base + wq_off); + wqi = gc->vaddr + wq_off + GUC_DB_SIZE; /* Now fill in the 4-word work queue item */ wqi->header = WQ_TYPE_INORDER | @@ -521,9 +550,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc, wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine); wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT; - wqi->fence_id = rq->fence.seqno; - - kunmap_atomic(base); + wqi->fence_id = rq->global_seqno; } static int guc_ring_doorbell(struct i915_guc_client *gc) @@ -533,7 +560,7 @@ static int guc_ring_doorbell(struct i915_guc_client *gc) union guc_doorbell_qw *db; int attempt = 2, ret = -EAGAIN; - desc = gc->client_base + gc->proc_desc_offset; + desc = gc->vaddr + gc->proc_desc_offset; /* Update the tail so it is visible to GuC */ desc->tail = gc->wq_tail; @@ -549,7 +576,7 @@ static int guc_ring_doorbell(struct i915_guc_client *gc) db_exc.cookie = 1; /* pointer of current doorbell cacheline */ - db = gc->client_base + gc->doorbell_offset; + db = gc->vaddr + gc->doorbell_offset; while (attempt--) { /* lets ring the doorbell */ @@ -601,13 +628,31 @@ static int guc_ring_doorbell(struct i915_guc_client *gc) */ static void i915_guc_submit(struct drm_i915_gem_request *rq) { - unsigned int engine_id = rq->engine->id; + struct drm_i915_private *dev_priv = rq->i915; + struct intel_engine_cs *engine = rq->engine; + unsigned int engine_id = engine->id; struct intel_guc *guc = &rq->i915->guc; struct i915_guc_client *client = guc->execbuf_client; int b_ret; + /* We keep the previous context alive until we retire the following + * request. This ensures that any the context object is still pinned + * for any residual writes the HW makes into it on the context switch + * into the next object following the breadcrumb. Otherwise, we may + * retire the context too early. + */ + rq->previous_context = engine->last_context; + engine->last_context = rq->ctx; + + i915_gem_request_submit(rq); + spin_lock(&client->wq_lock); guc_wq_item_append(client, rq); + + /* WA to flush out the pending GMADR writes to ring buffer. */ + if (i915_vma_is_map_and_fenceable(rq->ring->vma)) + POSTING_READ_FW(GUC_STATUS); + b_ret = guc_ring_doorbell(client); client->submissions[engine_id] += 1; @@ -616,7 +661,7 @@ static void i915_guc_submit(struct drm_i915_gem_request *rq) client->b_fail += 1; guc->submissions[engine_id] += 1; - guc->last_seqno[engine_id] = rq->fence.seqno; + guc->last_seqno[engine_id] = rq->global_seqno; spin_unlock(&client->wq_lock); } @@ -685,14 +730,14 @@ guc_client_free(struct drm_i915_private *dev_priv, * Be sure to drop any locks */ - if (client->client_base) { + if (client->vaddr) { /* * If we got as far as setting up a doorbell, make sure we * shut it down before unmapping & deallocating the memory. */ guc_disable_doorbell(guc, client); - kunmap(kmap_to_page(client->client_base)); + i915_gem_object_unpin_map(client->vma->obj); } i915_vma_unpin_and_release(&client->vma); @@ -781,6 +826,7 @@ guc_client_alloc(struct drm_i915_private *dev_priv, struct i915_guc_client *client; struct intel_guc *guc = &dev_priv->guc; struct i915_vma *vma; + void *vaddr; uint16_t db_id; client = kzalloc(sizeof(*client), GFP_KERNEL); @@ -807,7 +853,12 @@ guc_client_alloc(struct drm_i915_private *dev_priv, /* We'll keep just the first (doorbell/proc) page permanently kmap'd. */ client->vma = vma; - client->client_base = kmap(i915_vma_first_page(vma)); + + vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB); + if (IS_ERR(vaddr)) + goto err; + + client->vaddr = vaddr; spin_lock_init(&client->wq_lock); client->wq_offset = GUC_DB_SIZE; @@ -847,15 +898,411 @@ err: return NULL; } +/* + * Sub buffer switch callback. Called whenever relay has to switch to a new + * sub buffer, relay stays on the same sub buffer if 0 is returned. + */ +static int subbuf_start_callback(struct rchan_buf *buf, + void *subbuf, + void *prev_subbuf, + size_t prev_padding) +{ + /* Use no-overwrite mode by default, where relay will stop accepting + * new data if there are no empty sub buffers left. + * There is no strict synchronization enforced by relay between Consumer + * and Producer. In overwrite mode, there is a possibility of getting + * inconsistent/garbled data, the producer could be writing on to the + * same sub buffer from which Consumer is reading. This can't be avoided + * unless Consumer is fast enough and can always run in tandem with + * Producer. + */ + if (relay_buf_full(buf)) + return 0; + + return 1; +} + +/* + * file_create() callback. Creates relay file in debugfs. + */ +static struct dentry *create_buf_file_callback(const char *filename, + struct dentry *parent, + umode_t mode, + struct rchan_buf *buf, + int *is_global) +{ + struct dentry *buf_file; + + /* This to enable the use of a single buffer for the relay channel and + * correspondingly have a single file exposed to User, through which + * it can collect the logs in order without any post-processing. + * Need to set 'is_global' even if parent is NULL for early logging. + */ + *is_global = 1; + + if (!parent) + return NULL; + + /* Not using the channel filename passed as an argument, since for each + * channel relay appends the corresponding CPU number to the filename + * passed in relay_open(). This should be fine as relay just needs a + * dentry of the file associated with the channel buffer and that file's + * name need not be same as the filename passed as an argument. + */ + buf_file = debugfs_create_file("guc_log", mode, + parent, buf, &relay_file_operations); + return buf_file; +} + +/* + * file_remove() default callback. Removes relay file in debugfs. + */ +static int remove_buf_file_callback(struct dentry *dentry) +{ + debugfs_remove(dentry); + return 0; +} + +/* relay channel callbacks */ +static struct rchan_callbacks relay_callbacks = { + .subbuf_start = subbuf_start_callback, + .create_buf_file = create_buf_file_callback, + .remove_buf_file = remove_buf_file_callback, +}; + +static void guc_log_remove_relay_file(struct intel_guc *guc) +{ + relay_close(guc->log.relay_chan); +} + +static int guc_log_create_relay_channel(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct rchan *guc_log_relay_chan; + size_t n_subbufs, subbuf_size; + + /* Keep the size of sub buffers same as shared log buffer */ + subbuf_size = guc->log.vma->obj->base.size; + + /* Store up to 8 snapshots, which is large enough to buffer sufficient + * boot time logs and provides enough leeway to User, in terms of + * latency, for consuming the logs from relay. Also doesn't take + * up too much memory. + */ + n_subbufs = 8; + + guc_log_relay_chan = relay_open(NULL, NULL, subbuf_size, + n_subbufs, &relay_callbacks, dev_priv); + if (!guc_log_relay_chan) { + DRM_ERROR("Couldn't create relay chan for GuC logging\n"); + return -ENOMEM; + } + + GEM_BUG_ON(guc_log_relay_chan->subbuf_size < subbuf_size); + guc->log.relay_chan = guc_log_relay_chan; + return 0; +} + +static int guc_log_create_relay_file(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct dentry *log_dir; + int ret; + + /* For now create the log file in /sys/kernel/debug/dri/0 dir */ + log_dir = dev_priv->drm.primary->debugfs_root; + + /* If /sys/kernel/debug/dri/0 location do not exist, then debugfs is + * not mounted and so can't create the relay file. + * The relay API seems to fit well with debugfs only, for availing relay + * there are 3 requirements which can be met for debugfs file only in a + * straightforward/clean manner :- + * i) Need the associated dentry pointer of the file, while opening the + * relay channel. + * ii) Should be able to use 'relay_file_operations' fops for the file. + * iii) Set the 'i_private' field of file's inode to the pointer of + * relay channel buffer. + */ + if (!log_dir) { + DRM_ERROR("Debugfs dir not available yet for GuC log file\n"); + return -ENODEV; + } + + ret = relay_late_setup_files(guc->log.relay_chan, "guc_log", log_dir); + if (ret) { + DRM_ERROR("Couldn't associate relay chan with file %d\n", ret); + return ret; + } + + return 0; +} + +static void guc_move_to_next_buf(struct intel_guc *guc) +{ + /* Make sure the updates made in the sub buffer are visible when + * Consumer sees the following update to offset inside the sub buffer. + */ + smp_wmb(); + + /* All data has been written, so now move the offset of sub buffer. */ + relay_reserve(guc->log.relay_chan, guc->log.vma->obj->base.size); + + /* Switch to the next sub buffer */ + relay_flush(guc->log.relay_chan); +} + +static void *guc_get_write_buffer(struct intel_guc *guc) +{ + if (!guc->log.relay_chan) + return NULL; + + /* Just get the base address of a new sub buffer and copy data into it + * ourselves. NULL will be returned in no-overwrite mode, if all sub + * buffers are full. Could have used the relay_write() to indirectly + * copy the data, but that would have been bit convoluted, as we need to + * write to only certain locations inside a sub buffer which cannot be + * done without using relay_reserve() along with relay_write(). So its + * better to use relay_reserve() alone. + */ + return relay_reserve(guc->log.relay_chan, 0); +} + +static bool +guc_check_log_buf_overflow(struct intel_guc *guc, + enum guc_log_buffer_type type, unsigned int full_cnt) +{ + unsigned int prev_full_cnt = guc->log.prev_overflow_count[type]; + bool overflow = false; + + if (full_cnt != prev_full_cnt) { + overflow = true; + + guc->log.prev_overflow_count[type] = full_cnt; + guc->log.total_overflow_count[type] += full_cnt - prev_full_cnt; + + if (full_cnt < prev_full_cnt) { + /* buffer_full_cnt is a 4 bit counter */ + guc->log.total_overflow_count[type] += 16; + } + DRM_ERROR_RATELIMITED("GuC log buffer overflow\n"); + } + + return overflow; +} + +static unsigned int guc_get_log_buffer_size(enum guc_log_buffer_type type) +{ + switch (type) { + case GUC_ISR_LOG_BUFFER: + return (GUC_LOG_ISR_PAGES + 1) * PAGE_SIZE; + case GUC_DPC_LOG_BUFFER: + return (GUC_LOG_DPC_PAGES + 1) * PAGE_SIZE; + case GUC_CRASH_DUMP_LOG_BUFFER: + return (GUC_LOG_CRASH_PAGES + 1) * PAGE_SIZE; + default: + MISSING_CASE(type); + } + + return 0; +} + +static void guc_read_update_log_buffer(struct intel_guc *guc) +{ + unsigned int buffer_size, read_offset, write_offset, bytes_to_copy, full_cnt; + struct guc_log_buffer_state *log_buf_state, *log_buf_snapshot_state; + struct guc_log_buffer_state log_buf_state_local; + enum guc_log_buffer_type type; + void *src_data, *dst_data; + bool new_overflow; + + if (WARN_ON(!guc->log.buf_addr)) + return; + + /* Get the pointer to shared GuC log buffer */ + log_buf_state = src_data = guc->log.buf_addr; + + /* Get the pointer to local buffer to store the logs */ + log_buf_snapshot_state = dst_data = guc_get_write_buffer(guc); + + /* Actual logs are present from the 2nd page */ + src_data += PAGE_SIZE; + dst_data += PAGE_SIZE; + + for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { + /* Make a copy of the state structure, inside GuC log buffer + * (which is uncached mapped), on the stack to avoid reading + * from it multiple times. + */ + memcpy(&log_buf_state_local, log_buf_state, + sizeof(struct guc_log_buffer_state)); + buffer_size = guc_get_log_buffer_size(type); + read_offset = log_buf_state_local.read_ptr; + write_offset = log_buf_state_local.sampled_write_ptr; + full_cnt = log_buf_state_local.buffer_full_cnt; + + /* Bookkeeping stuff */ + guc->log.flush_count[type] += log_buf_state_local.flush_to_file; + new_overflow = guc_check_log_buf_overflow(guc, type, full_cnt); + + /* Update the state of shared log buffer */ + log_buf_state->read_ptr = write_offset; + log_buf_state->flush_to_file = 0; + log_buf_state++; + + if (unlikely(!log_buf_snapshot_state)) + continue; + + /* First copy the state structure in snapshot buffer */ + memcpy(log_buf_snapshot_state, &log_buf_state_local, + sizeof(struct guc_log_buffer_state)); + + /* The write pointer could have been updated by GuC firmware, + * after sending the flush interrupt to Host, for consistency + * set write pointer value to same value of sampled_write_ptr + * in the snapshot buffer. + */ + log_buf_snapshot_state->write_ptr = write_offset; + log_buf_snapshot_state++; + + /* Now copy the actual logs. */ + if (unlikely(new_overflow)) { + /* copy the whole buffer in case of overflow */ + read_offset = 0; + write_offset = buffer_size; + } else if (unlikely((read_offset > buffer_size) || + (write_offset > buffer_size))) { + DRM_ERROR("invalid log buffer state\n"); + /* copy whole buffer as offsets are unreliable */ + read_offset = 0; + write_offset = buffer_size; + } + + /* Just copy the newly written data */ + if (read_offset > write_offset) { + i915_memcpy_from_wc(dst_data, src_data, write_offset); + bytes_to_copy = buffer_size - read_offset; + } else { + bytes_to_copy = write_offset - read_offset; + } + i915_memcpy_from_wc(dst_data + read_offset, + src_data + read_offset, bytes_to_copy); + + src_data += buffer_size; + dst_data += buffer_size; + } + + if (log_buf_snapshot_state) + guc_move_to_next_buf(guc); + else { + /* Used rate limited to avoid deluge of messages, logs might be + * getting consumed by User at a slow rate. + */ + DRM_ERROR_RATELIMITED("no sub-buffer to capture logs\n"); + guc->log.capture_miss_count++; + } +} + +static void guc_capture_logs_work(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, struct drm_i915_private, guc.log.flush_work); + + i915_guc_capture_logs(dev_priv); +} + +static void guc_log_cleanup(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + /* First disable the flush interrupt */ + gen9_disable_guc_interrupts(dev_priv); + + if (guc->log.flush_wq) + destroy_workqueue(guc->log.flush_wq); + + guc->log.flush_wq = NULL; + + if (guc->log.relay_chan) + guc_log_remove_relay_file(guc); + + guc->log.relay_chan = NULL; + + if (guc->log.buf_addr) + i915_gem_object_unpin_map(guc->log.vma->obj); + + guc->log.buf_addr = NULL; +} + +static int guc_log_create_extras(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + void *vaddr; + int ret; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + /* Nothing to do */ + if (i915.guc_log_level < 0) + return 0; + + if (!guc->log.buf_addr) { + /* Create a WC (Uncached for read) vmalloc mapping of log + * buffer pages, so that we can directly get the data + * (up-to-date) from memory. + */ + vaddr = i915_gem_object_pin_map(guc->log.vma->obj, I915_MAP_WC); + if (IS_ERR(vaddr)) { + ret = PTR_ERR(vaddr); + DRM_ERROR("Couldn't map log buffer pages %d\n", ret); + return ret; + } + + guc->log.buf_addr = vaddr; + } + + if (!guc->log.relay_chan) { + /* Create a relay channel, so that we have buffers for storing + * the GuC firmware logs, the channel will be linked with a file + * later on when debugfs is registered. + */ + ret = guc_log_create_relay_channel(guc); + if (ret) + return ret; + } + + if (!guc->log.flush_wq) { + INIT_WORK(&guc->log.flush_work, guc_capture_logs_work); + + /* + * GuC log buffer flush work item has to do register access to + * send the ack to GuC and this work item, if not synced before + * suspend, can potentially get executed after the GFX device is + * suspended. + * By marking the WQ as freezable, we don't have to bother about + * flushing of this work item from the suspend hooks, the pending + * work item if any will be either executed before the suspend + * or scheduled later on resume. This way the handling of work + * item can be kept same between system suspend & rpm suspend. + */ + guc->log.flush_wq = alloc_ordered_workqueue("i915-guc_log", + WQ_HIGHPRI | WQ_FREEZABLE); + if (guc->log.flush_wq == NULL) { + DRM_ERROR("Couldn't allocate the wq for GuC logging\n"); + return -ENOMEM; + } + } + + return 0; +} + static void guc_log_create(struct intel_guc *guc) { struct i915_vma *vma; unsigned long offset; uint32_t size, flags; - if (i915.guc_log_level < GUC_LOG_VERBOSITY_MIN) - return; - if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX) i915.guc_log_level = GUC_LOG_VERBOSITY_MAX; @@ -865,8 +1312,18 @@ static void guc_log_create(struct intel_guc *guc) GUC_LOG_ISR_PAGES + 1 + GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT; - vma = guc->log_vma; + vma = guc->log.vma; if (!vma) { + /* We require SSE 4.1 for fast reads from the GuC log buffer and + * it should be present on the chipsets supporting GuC based + * submisssions. + */ + if (WARN_ON(!i915_memcpy_from_wc(NULL, NULL, 0))) { + /* logging will not be enabled */ + i915.guc_log_level = -1; + return; + } + vma = guc_allocate_vma(guc, size); if (IS_ERR(vma)) { /* logging will be off */ @@ -874,7 +1331,14 @@ static void guc_log_create(struct intel_guc *guc) return; } - guc->log_vma = vma; + guc->log.vma = vma; + + if (guc_log_create_extras(guc)) { + guc_log_cleanup(guc); + i915_vma_unpin_and_release(&guc->log.vma); + i915.guc_log_level = -1; + return; + } } /* each allocated unit is a page */ @@ -884,7 +1348,37 @@ static void guc_log_create(struct intel_guc *guc) (GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT); offset = i915_ggtt_offset(vma) >> PAGE_SHIFT; /* in pages */ - guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; + guc->log.flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; +} + +static int guc_log_late_setup(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + int ret; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + if (i915.guc_log_level < 0) + return -EINVAL; + + /* If log_level was set as -1 at boot time, then setup needed to + * handle log buffer flush interrupts would not have been done yet, + * so do that now. + */ + ret = guc_log_create_extras(guc); + if (ret) + goto err; + + ret = guc_log_create_relay_file(guc); + if (ret) + goto err; + + return 0; +err: + guc_log_cleanup(guc); + /* logging will remain off */ + i915.guc_log_level = -1; + return ret; } static void guc_policies_init(struct guc_policies *policies) @@ -1006,6 +1500,7 @@ int i915_guc_submission_init(struct drm_i915_private *dev_priv) guc->ctx_pool_vma = vma; ida_init(&guc->ctx_ids); + mutex_init(&guc->action_lock); guc_log_create(guc); guc_addon_create(guc); @@ -1037,9 +1532,11 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv) /* Take over from manual control of ELSP (execlists) */ for_each_engine(engine, dev_priv, id) { engine->submit_request = i915_guc_submit; + engine->schedule = NULL; /* Replay the current set of previously submitted requests */ - list_for_each_entry(request, &engine->request_list, link) { + list_for_each_entry(request, + &engine->timeline->requests, link) { client->wq_rsvd += sizeof(struct guc_wq_item); if (i915_sw_fence_done(&request->submit)) i915_guc_submit(request); @@ -1068,7 +1565,7 @@ void i915_guc_submission_fini(struct drm_i915_private *dev_priv) struct intel_guc *guc = &dev_priv->guc; i915_vma_unpin_and_release(&guc->ads_vma); - i915_vma_unpin_and_release(&guc->log_vma); + i915_vma_unpin_and_release(&guc->log.vma); if (guc->ctx_pool_vma) ida_destroy(&guc->ctx_ids); @@ -1089,6 +1586,8 @@ int intel_guc_suspend(struct drm_device *dev) if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS) return 0; + gen9_disable_guc_interrupts(dev_priv); + ctx = dev_priv->kernel_context; data[0] = HOST2GUC_ACTION_ENTER_S_STATE; @@ -1115,6 +1614,9 @@ int intel_guc_resume(struct drm_device *dev) if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS) return 0; + if (i915.guc_log_level >= 0) + gen9_enable_guc_interrupts(dev_priv); + ctx = dev_priv->kernel_context; data[0] = HOST2GUC_ACTION_EXIT_S_STATE; @@ -1124,3 +1626,104 @@ int intel_guc_resume(struct drm_device *dev) return host2guc_action(guc, data, ARRAY_SIZE(data)); } + +void i915_guc_capture_logs(struct drm_i915_private *dev_priv) +{ + guc_read_update_log_buffer(&dev_priv->guc); + + /* Generally device is expected to be active only at this + * time, so get/put should be really quick. + */ + intel_runtime_pm_get(dev_priv); + host2guc_logbuffer_flush_complete(&dev_priv->guc); + intel_runtime_pm_put(dev_priv); +} + +void i915_guc_flush_logs(struct drm_i915_private *dev_priv) +{ + if (!i915.enable_guc_submission || (i915.guc_log_level < 0)) + return; + + /* First disable the interrupts, will be renabled afterwards */ + gen9_disable_guc_interrupts(dev_priv); + + /* Before initiating the forceful flush, wait for any pending/ongoing + * flush to complete otherwise forceful flush may not actually happen. + */ + flush_work(&dev_priv->guc.log.flush_work); + + /* Ask GuC to update the log buffer state */ + host2guc_force_logbuffer_flush(&dev_priv->guc); + + /* GuC would have updated log buffer by now, so capture it */ + i915_guc_capture_logs(dev_priv); +} + +void i915_guc_unregister(struct drm_i915_private *dev_priv) +{ + if (!i915.enable_guc_submission) + return; + + mutex_lock(&dev_priv->drm.struct_mutex); + guc_log_cleanup(&dev_priv->guc); + mutex_unlock(&dev_priv->drm.struct_mutex); +} + +void i915_guc_register(struct drm_i915_private *dev_priv) +{ + if (!i915.enable_guc_submission) + return; + + mutex_lock(&dev_priv->drm.struct_mutex); + guc_log_late_setup(&dev_priv->guc); + mutex_unlock(&dev_priv->drm.struct_mutex); +} + +int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val) +{ + union guc_log_control log_param; + int ret; + + log_param.value = control_val; + + if (log_param.verbosity < GUC_LOG_VERBOSITY_MIN || + log_param.verbosity > GUC_LOG_VERBOSITY_MAX) + return -EINVAL; + + /* This combination doesn't make sense & won't have any effect */ + if (!log_param.logging_enabled && (i915.guc_log_level < 0)) + return 0; + + ret = host2guc_logging_control(&dev_priv->guc, log_param.value); + if (ret < 0) { + DRM_DEBUG_DRIVER("host2guc action failed %d\n", ret); + return ret; + } + + i915.guc_log_level = log_param.verbosity; + + /* If log_level was set as -1 at boot time, then the relay channel file + * wouldn't have been created by now and interrupts also would not have + * been enabled. + */ + if (!dev_priv->guc.log.relay_chan) { + ret = guc_log_late_setup(&dev_priv->guc); + if (!ret) + gen9_enable_guc_interrupts(dev_priv); + } else if (!log_param.logging_enabled) { + /* Once logging is disabled, GuC won't generate logs & send an + * interrupt. But there could be some data in the log buffer + * which is yet to be captured. So request GuC to update the log + * buffer state and then collect the left over logs. + */ + i915_guc_flush_logs(dev_priv); + + /* As logging is disabled, update log level to reflect that */ + i915.guc_log_level = -1; + } else { + /* In case interrupts were disabled, enable them now */ + gen9_enable_guc_interrupts(dev_priv); + } + + return ret; +} diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 23315e5461bf..07ca71cabb2b 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -170,6 +170,7 @@ static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, } while (0) static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); +static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); /* For display hotplug interrupt */ static inline void @@ -303,18 +304,18 @@ static void snb_update_pm_irq(struct drm_i915_private *dev_priv, assert_spin_locked(&dev_priv->irq_lock); - new_val = dev_priv->pm_irq_mask; + new_val = dev_priv->pm_imr; new_val &= ~interrupt_mask; new_val |= (~enabled_irq_mask & interrupt_mask); - if (new_val != dev_priv->pm_irq_mask) { - dev_priv->pm_irq_mask = new_val; - I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask); + if (new_val != dev_priv->pm_imr) { + dev_priv->pm_imr = new_val; + I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr); POSTING_READ(gen6_pm_imr(dev_priv)); } } -void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) +void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) { if (WARN_ON(!intel_irqs_enabled(dev_priv))) return; @@ -322,28 +323,54 @@ void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) snb_update_pm_irq(dev_priv, mask, mask); } -static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv, - uint32_t mask) +static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) { snb_update_pm_irq(dev_priv, mask, 0); } -void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) +void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) { if (WARN_ON(!intel_irqs_enabled(dev_priv))) return; - __gen6_disable_pm_irq(dev_priv, mask); + __gen6_mask_pm_irq(dev_priv, mask); } -void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) +void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask) { i915_reg_t reg = gen6_pm_iir(dev_priv); - spin_lock_irq(&dev_priv->irq_lock); - I915_WRITE(reg, dev_priv->pm_rps_events); - I915_WRITE(reg, dev_priv->pm_rps_events); + assert_spin_locked(&dev_priv->irq_lock); + + I915_WRITE(reg, reset_mask); + I915_WRITE(reg, reset_mask); POSTING_READ(reg); +} + +void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask) +{ + assert_spin_locked(&dev_priv->irq_lock); + + dev_priv->pm_ier |= enable_mask; + I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier); + gen6_unmask_pm_irq(dev_priv, enable_mask); + /* unmask_pm_irq provides an implicit barrier (POSTING_READ) */ +} + +void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask) +{ + assert_spin_locked(&dev_priv->irq_lock); + + dev_priv->pm_ier &= ~disable_mask; + __gen6_mask_pm_irq(dev_priv, disable_mask); + I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier); + /* though a barrier is missing here, but don't really need a one */ +} + +void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events); dev_priv->rps.pm_iir = 0; spin_unlock_irq(&dev_priv->irq_lock); } @@ -357,8 +384,6 @@ void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv) WARN_ON_ONCE(dev_priv->rps.pm_iir); WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); dev_priv->rps.interrupts_enabled = true; - I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) | - dev_priv->pm_rps_events); gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); spin_unlock_irq(&dev_priv->irq_lock); @@ -379,9 +404,7 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u)); - __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events); - I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) & - ~dev_priv->pm_rps_events); + gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events); spin_unlock_irq(&dev_priv->irq_lock); synchronize_irq(dev_priv->drm.irq); @@ -395,6 +418,38 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) gen6_reset_rps_interrupts(dev_priv); } +void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events); + spin_unlock_irq(&dev_priv->irq_lock); +} + +void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + if (!dev_priv->guc.interrupts_enabled) { + WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & + dev_priv->pm_guc_events); + dev_priv->guc.interrupts_enabled = true; + gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events); + } + spin_unlock_irq(&dev_priv->irq_lock); +} + +void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + dev_priv->guc.interrupts_enabled = false; + + gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events); + + spin_unlock_irq(&dev_priv->irq_lock); + synchronize_irq(dev_priv->drm.irq); + + gen9_reset_guc_interrupts(dev_priv); +} + /** * bdw_update_port_irq - update DE port interrupt * @dev_priv: driver private @@ -670,8 +725,8 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t high_frame, low_frame; u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; - struct intel_crtc *intel_crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); const struct drm_display_mode *mode = &intel_crtc->base.hwmode; htotal = mode->crtc_htotal; @@ -776,8 +831,8 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, const struct drm_display_mode *mode) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; bool in_vbl = true; @@ -912,21 +967,22 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, struct timeval *vblank_time, unsigned flags) { - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc; - if (pipe >= INTEL_INFO(dev)->num_pipes) { + if (pipe >= INTEL_INFO(dev_priv)->num_pipes) { DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } /* Get drm_crtc to timestamp: */ - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc == NULL) { DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } - if (!crtc->hwmode.crtc_clock) { + if (!crtc->base.hwmode.crtc_clock) { DRM_DEBUG_KMS("crtc %u is disabled\n", pipe); return -EBUSY; } @@ -934,7 +990,7 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, /* Helper routine in DRM core does all the work: */ return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, vblank_time, flags, - &crtc->hwmode); + &crtc->base.hwmode); } static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) @@ -1085,7 +1141,7 @@ static void gen6_pm_rps_work(struct work_struct *work) pm_iir = dev_priv->rps.pm_iir; dev_priv->rps.pm_iir = 0; /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ - gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); + gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events); client_boost = dev_priv->rps.client_boost; dev_priv->rps.client_boost = false; spin_unlock_irq(&dev_priv->irq_lock); @@ -1324,11 +1380,13 @@ static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv, DRM_ERROR("The master control interrupt lied (GT3)!\n"); } - if (master_ctl & GEN8_GT_PM_IRQ) { + if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2)); - if (gt_iir[2] & dev_priv->pm_rps_events) { + if (gt_iir[2] & (dev_priv->pm_rps_events | + dev_priv->pm_guc_events)) { I915_WRITE_FW(GEN8_GT_IIR(2), - gt_iir[2] & dev_priv->pm_rps_events); + gt_iir[2] & (dev_priv->pm_rps_events | + dev_priv->pm_guc_events)); ret = IRQ_HANDLED; } else DRM_ERROR("The master control interrupt lied (PM)!\n"); @@ -1360,6 +1418,9 @@ static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv, if (gt_iir[2] & dev_priv->pm_rps_events) gen6_rps_irq_handler(dev_priv, gt_iir[2]); + + if (gt_iir[2] & dev_priv->pm_guc_events) + gen9_guc_irq_handler(dev_priv, gt_iir[2]); } static bool bxt_port_hotplug_long_detect(enum port port, u32 val) @@ -1586,7 +1647,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) { if (pm_iir & dev_priv->pm_rps_events) { spin_lock(&dev_priv->irq_lock); - gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); + gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); if (dev_priv->rps.interrupts_enabled) { dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events; schedule_work(&dev_priv->rps.work); @@ -1606,6 +1667,41 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) } } +static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) +{ + if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) { + /* Sample the log buffer flush related bits & clear them out now + * itself from the message identity register to minimize the + * probability of losing a flush interrupt, when there are back + * to back flush interrupts. + * There can be a new flush interrupt, for different log buffer + * type (like for ISR), whilst Host is handling one (for DPC). + * Since same bit is used in message register for ISR & DPC, it + * could happen that GuC sets the bit for 2nd interrupt but Host + * clears out the bit on handling the 1st interrupt. + */ + u32 msg, flush; + + msg = I915_READ(SOFT_SCRATCH(15)); + flush = msg & (GUC2HOST_MSG_CRASH_DUMP_POSTED | + GUC2HOST_MSG_FLUSH_LOG_BUFFER); + if (flush) { + /* Clear the message bits that are handled */ + I915_WRITE(SOFT_SCRATCH(15), msg & ~flush); + + /* Handle flush interrupt in bottom half */ + queue_work(dev_priv->guc.log.flush_wq, + &dev_priv->guc.log.flush_work); + + dev_priv->guc.log.flush_interrupt_count++; + } else { + /* Not clearing of unhandled event bits won't result in + * re-triggering of the interrupt. + */ + } + } +} + static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) { @@ -2408,7 +2504,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS; if (fault_errors) - DRM_ERROR("Fault errors on pipe %c\n: 0x%08x", + DRM_ERROR("Fault errors on pipe %c: 0x%08x\n", pipe_name(pipe), fault_errors); } @@ -2752,424 +2848,8 @@ static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe) spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } -static bool -ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr) -{ - if (INTEL_GEN(engine->i915) >= 8) { - return (ipehr >> 23) == 0x1c; - } else { - ipehr &= ~MI_SEMAPHORE_SYNC_MASK; - return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | - MI_SEMAPHORE_REGISTER); - } -} - -static struct intel_engine_cs * -semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr, - u64 offset) -{ - struct drm_i915_private *dev_priv = engine->i915; - struct intel_engine_cs *signaller; - enum intel_engine_id id; - - if (INTEL_GEN(dev_priv) >= 8) { - for_each_engine(signaller, dev_priv, id) { - if (engine == signaller) - continue; - - if (offset == signaller->semaphore.signal_ggtt[engine->hw_id]) - return signaller; - } - } else { - u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; - - for_each_engine(signaller, dev_priv, id) { - if(engine == signaller) - continue; - - if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id]) - return signaller; - } - } - - DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n", - engine->name, ipehr, offset); - - return ERR_PTR(-ENODEV); -} - -static struct intel_engine_cs * -semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) -{ - struct drm_i915_private *dev_priv = engine->i915; - void __iomem *vaddr; - u32 cmd, ipehr, head; - u64 offset = 0; - int i, backwards; - - /* - * This function does not support execlist mode - any attempt to - * proceed further into this function will result in a kernel panic - * when dereferencing ring->buffer, which is not set up in execlist - * mode. - * - * The correct way of doing it would be to derive the currently - * executing ring buffer from the current context, which is derived - * from the currently running request. Unfortunately, to get the - * current request we would have to grab the struct_mutex before doing - * anything else, which would be ill-advised since some other thread - * might have grabbed it already and managed to hang itself, causing - * the hang checker to deadlock. - * - * Therefore, this function does not support execlist mode in its - * current form. Just return NULL and move on. - */ - if (engine->buffer == NULL) - return NULL; - - ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); - if (!ipehr_is_semaphore_wait(engine, ipehr)) - return NULL; - - /* - * HEAD is likely pointing to the dword after the actual command, - * so scan backwards until we find the MBOX. But limit it to just 3 - * or 4 dwords depending on the semaphore wait command size. - * Note that we don't care about ACTHD here since that might - * point at at batch, and semaphores are always emitted into the - * ringbuffer itself. - */ - head = I915_READ_HEAD(engine) & HEAD_ADDR; - backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4; - vaddr = (void __iomem *)engine->buffer->vaddr; - - for (i = backwards; i; --i) { - /* - * Be paranoid and presume the hw has gone off into the wild - - * our ring is smaller than what the hardware (and hence - * HEAD_ADDR) allows. Also handles wrap-around. - */ - head &= engine->buffer->size - 1; - - /* This here seems to blow up */ - cmd = ioread32(vaddr + head); - if (cmd == ipehr) - break; - - head -= 4; - } - - if (!i) - return NULL; - - *seqno = ioread32(vaddr + head + 4) + 1; - if (INTEL_GEN(dev_priv) >= 8) { - offset = ioread32(vaddr + head + 12); - offset <<= 32; - offset |= ioread32(vaddr + head + 8); - } - return semaphore_wait_to_signaller_ring(engine, ipehr, offset); -} - -static int semaphore_passed(struct intel_engine_cs *engine) +static void ibx_irq_reset(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = engine->i915; - struct intel_engine_cs *signaller; - u32 seqno; - - engine->hangcheck.deadlock++; - - signaller = semaphore_waits_for(engine, &seqno); - if (signaller == NULL) - return -1; - - if (IS_ERR(signaller)) - return 0; - - /* Prevent pathological recursion due to driver bugs */ - if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES) - return -1; - - if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno)) - return 1; - - /* cursory check for an unkickable deadlock */ - if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE && - semaphore_passed(signaller) < 0) - return -1; - - return 0; -} - -static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, dev_priv, id) - engine->hangcheck.deadlock = 0; -} - -static bool instdone_unchanged(u32 current_instdone, u32 *old_instdone) -{ - u32 tmp = current_instdone | *old_instdone; - bool unchanged; - - unchanged = tmp == *old_instdone; - *old_instdone |= tmp; - - return unchanged; -} - -static bool subunits_stuck(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - struct intel_instdone instdone; - struct intel_instdone *accu_instdone = &engine->hangcheck.instdone; - bool stuck; - int slice; - int subslice; - - if (engine->id != RCS) - return true; - - intel_engine_get_instdone(engine, &instdone); - - /* There might be unstable subunit states even when - * actual head is not moving. Filter out the unstable ones by - * accumulating the undone -> done transitions and only - * consider those as progress. - */ - stuck = instdone_unchanged(instdone.instdone, - &accu_instdone->instdone); - stuck &= instdone_unchanged(instdone.slice_common, - &accu_instdone->slice_common); - - for_each_instdone_slice_subslice(dev_priv, slice, subslice) { - stuck &= instdone_unchanged(instdone.sampler[slice][subslice], - &accu_instdone->sampler[slice][subslice]); - stuck &= instdone_unchanged(instdone.row[slice][subslice], - &accu_instdone->row[slice][subslice]); - } - - return stuck; -} - -static enum intel_engine_hangcheck_action -head_stuck(struct intel_engine_cs *engine, u64 acthd) -{ - if (acthd != engine->hangcheck.acthd) { - - /* Clear subunit states on head movement */ - memset(&engine->hangcheck.instdone, 0, - sizeof(engine->hangcheck.instdone)); - - return HANGCHECK_ACTIVE; - } - - if (!subunits_stuck(engine)) - return HANGCHECK_ACTIVE; - - return HANGCHECK_HUNG; -} - -static enum intel_engine_hangcheck_action -engine_stuck(struct intel_engine_cs *engine, u64 acthd) -{ - struct drm_i915_private *dev_priv = engine->i915; - enum intel_engine_hangcheck_action ha; - u32 tmp; - - ha = head_stuck(engine, acthd); - if (ha != HANGCHECK_HUNG) - return ha; - - if (IS_GEN2(dev_priv)) - return HANGCHECK_HUNG; - - /* Is the chip hanging on a WAIT_FOR_EVENT? - * If so we can simply poke the RB_WAIT bit - * and break the hang. This should work on - * all but the second generation chipsets. - */ - tmp = I915_READ_CTL(engine); - if (tmp & RING_WAIT) { - i915_handle_error(dev_priv, 0, - "Kicking stuck wait on %s", - engine->name); - I915_WRITE_CTL(engine, tmp); - return HANGCHECK_KICK; - } - - if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) { - switch (semaphore_passed(engine)) { - default: - return HANGCHECK_HUNG; - case 1: - i915_handle_error(dev_priv, 0, - "Kicking stuck semaphore on %s", - engine->name); - I915_WRITE_CTL(engine, tmp); - return HANGCHECK_KICK; - case 0: - return HANGCHECK_WAIT; - } - } - - return HANGCHECK_HUNG; -} - -/* - * This is called when the chip hasn't reported back with completed - * batchbuffers in a long time. We keep track per ring seqno progress and - * if there are no progress, hangcheck score for that ring is increased. - * Further, acthd is inspected to see if the ring is stuck. On stuck case - * we kick the ring. If we see no progress on three subsequent calls - * we assume chip is wedged and try to fix it by resetting the chip. - */ -static void i915_hangcheck_elapsed(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), - gpu_error.hangcheck_work.work); - struct intel_engine_cs *engine; - enum intel_engine_id id; - unsigned int hung = 0, stuck = 0; - int busy_count = 0; -#define BUSY 1 -#define KICK 5 -#define HUNG 20 -#define ACTIVE_DECAY 15 - - if (!i915.enable_hangcheck) - return; - - if (!READ_ONCE(dev_priv->gt.awake)) - return; - - /* As enabling the GPU requires fairly extensive mmio access, - * periodically arm the mmio checker to see if we are triggering - * any invalid access. - */ - intel_uncore_arm_unclaimed_mmio_detection(dev_priv); - - for_each_engine(engine, dev_priv, id) { - bool busy = intel_engine_has_waiter(engine); - u64 acthd; - u32 seqno; - u32 submit; - - semaphore_clear_deadlocks(dev_priv); - - /* We don't strictly need an irq-barrier here, as we are not - * serving an interrupt request, be paranoid in case the - * barrier has side-effects (such as preventing a broken - * cacheline snoop) and so be sure that we can see the seqno - * advance. If the seqno should stick, due to a stale - * cacheline, we would erroneously declare the GPU hung. - */ - if (engine->irq_seqno_barrier) - engine->irq_seqno_barrier(engine); - - acthd = intel_engine_get_active_head(engine); - seqno = intel_engine_get_seqno(engine); - submit = READ_ONCE(engine->last_submitted_seqno); - - if (engine->hangcheck.seqno == seqno) { - if (i915_seqno_passed(seqno, submit)) { - engine->hangcheck.action = HANGCHECK_IDLE; - } else { - /* We always increment the hangcheck score - * if the engine is busy and still processing - * the same request, so that no single request - * can run indefinitely (such as a chain of - * batches). The only time we do not increment - * the hangcheck score on this ring, if this - * engine is in a legitimate wait for another - * engine. In that case the waiting engine is a - * victim and we want to be sure we catch the - * right culprit. Then every time we do kick - * the ring, add a small increment to the - * score so that we can catch a batch that is - * being repeatedly kicked and so responsible - * for stalling the machine. - */ - engine->hangcheck.action = - engine_stuck(engine, acthd); - - switch (engine->hangcheck.action) { - case HANGCHECK_IDLE: - case HANGCHECK_WAIT: - break; - case HANGCHECK_ACTIVE: - engine->hangcheck.score += BUSY; - break; - case HANGCHECK_KICK: - engine->hangcheck.score += KICK; - break; - case HANGCHECK_HUNG: - engine->hangcheck.score += HUNG; - break; - } - } - - if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { - hung |= intel_engine_flag(engine); - if (engine->hangcheck.action != HANGCHECK_HUNG) - stuck |= intel_engine_flag(engine); - } - } else { - engine->hangcheck.action = HANGCHECK_ACTIVE; - - /* Gradually reduce the count so that we catch DoS - * attempts across multiple batches. - */ - if (engine->hangcheck.score > 0) - engine->hangcheck.score -= ACTIVE_DECAY; - if (engine->hangcheck.score < 0) - engine->hangcheck.score = 0; - - /* Clear head and subunit states on seqno movement */ - acthd = 0; - - memset(&engine->hangcheck.instdone, 0, - sizeof(engine->hangcheck.instdone)); - } - - engine->hangcheck.seqno = seqno; - engine->hangcheck.acthd = acthd; - busy_count += busy; - } - - if (hung) { - char msg[80]; - unsigned int tmp; - int len; - - /* If some rings hung but others were still busy, only - * blame the hanging rings in the synopsis. - */ - if (stuck != hung) - hung &= ~stuck; - len = scnprintf(msg, sizeof(msg), - "%s on ", stuck == hung ? "No progress" : "Hang"); - for_each_engine_masked(engine, dev_priv, hung, tmp) - len += scnprintf(msg + len, sizeof(msg) - len, - "%s, ", engine->name); - msg[len-2] = '\0'; - - return i915_handle_error(dev_priv, hung, msg); - } - - /* Reset timer in case GPU hangs without another request being added */ - if (busy_count) - i915_queue_hangcheck(dev_priv); -} - -static void ibx_irq_reset(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - if (HAS_PCH_NOP(dev_priv)) return; @@ -3199,12 +2879,10 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev) POSTING_READ(SDEIER); } -static void gen5_gt_irq_reset(struct drm_device *dev) +static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - GEN5_IRQ_RESET(GT); - if (INTEL_INFO(dev)->gen >= 6) + if (INTEL_GEN(dev_priv) >= 6) GEN5_IRQ_RESET(GEN6_PM); } @@ -3269,9 +2947,9 @@ static void ironlake_irq_reset(struct drm_device *dev) if (IS_GEN7(dev_priv)) I915_WRITE(GEN7_ERR_INT, 0xffffffff); - gen5_gt_irq_reset(dev); + gen5_gt_irq_reset(dev_priv); - ibx_irq_reset(dev); + ibx_irq_reset(dev_priv); } static void valleyview_irq_preinstall(struct drm_device *dev) @@ -3281,7 +2959,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev) I915_WRITE(VLV_MASTER_IER, 0); POSTING_READ(VLV_MASTER_IER); - gen5_gt_irq_reset(dev); + gen5_gt_irq_reset(dev_priv); spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->display_irqs_enabled) @@ -3317,7 +2995,7 @@ static void gen8_irq_reset(struct drm_device *dev) GEN5_IRQ_RESET(GEN8_PCU_); if (HAS_PCH_SPLIT(dev_priv)) - ibx_irq_reset(dev); + ibx_irq_reset(dev_priv); } void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, @@ -3540,16 +3218,18 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev) GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs); - if (INTEL_INFO(dev)->gen >= 6) { + if (INTEL_GEN(dev_priv) >= 6) { /* * RPS interrupts will get enabled/disabled on demand when RPS * itself is enabled/disabled. */ - if (HAS_VEBOX(dev)) + if (HAS_VEBOX(dev_priv)) { pm_irqs |= PM_VEBOX_USER_INTERRUPT; + dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT; + } - dev_priv->pm_irq_mask = 0xffffffff; - GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs); + dev_priv->pm_imr = 0xffffffff; + GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs); } } @@ -3558,7 +3238,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev) struct drm_i915_private *dev_priv = to_i915(dev); u32 display_mask, extra_mask; - if (INTEL_INFO(dev)->gen >= 7) { + if (INTEL_GEN(dev_priv) >= 7) { display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB | DE_PLANEB_FLIP_DONE_IVB | @@ -3669,14 +3349,15 @@ static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) if (HAS_L3_DPF(dev_priv)) gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; - dev_priv->pm_irq_mask = 0xffffffff; + dev_priv->pm_ier = 0x0; + dev_priv->pm_imr = ~dev_priv->pm_ier; GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]); GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]); /* * RPS interrupts will get enabled/disabled on demand when RPS itself - * is enabled/disabled. + * is enabled/disabled. Same wil be the case for GuC interrupts. */ - GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0); + GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier); GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]); } @@ -3781,7 +3462,7 @@ static void valleyview_irq_uninstall(struct drm_device *dev) I915_WRITE(VLV_MASTER_IER, 0); POSTING_READ(VLV_MASTER_IER); - gen5_gt_irq_reset(dev); + gen5_gt_irq_reset(dev_priv); I915_WRITE(HWSTAM, 0xffffffff); @@ -3993,7 +3674,7 @@ static void i915_irq_preinstall(struct drm_device * dev) struct drm_i915_private *dev_priv = to_i915(dev); int pipe; - if (I915_HAS_HOTPLUG(dev)) { + if (I915_HAS_HOTPLUG(dev_priv)) { i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); } @@ -4027,7 +3708,7 @@ static int i915_irq_postinstall(struct drm_device *dev) I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | I915_USER_INTERRUPT; - if (I915_HAS_HOTPLUG(dev)) { + if (I915_HAS_HOTPLUG(dev_priv)) { i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); POSTING_READ(PORT_HOTPLUG_EN); @@ -4195,7 +3876,7 @@ static void i915_irq_uninstall(struct drm_device * dev) struct drm_i915_private *dev_priv = to_i915(dev); int pipe; - if (I915_HAS_HOTPLUG(dev)) { + if (I915_HAS_HOTPLUG(dev_priv)) { i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); } @@ -4460,6 +4141,9 @@ void intel_irq_init(struct drm_i915_private *dev_priv) INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); + if (HAS_GUC_SCHED(dev_priv)) + dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT; + /* Let's track the enabled rps events */ if (IS_VALLEYVIEW(dev_priv)) /* WaGsvRC0ResidencyMethod:vlv */ @@ -4481,9 +4165,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (INTEL_INFO(dev_priv)->gen >= 8) dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_GUC; - INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work, - i915_hangcheck_elapsed); - if (IS_GEN2(dev_priv)) { /* Gen2 doesn't have a hardware frame counter */ dev->max_vblank_count = 0; diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index 629e4334719c..d46ffe7086bc 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -39,7 +39,7 @@ struct i915_params i915 __read_mostly = { .enable_hangcheck = true, .enable_ppgtt = -1, .enable_psr = -1, - .preliminary_hw_support = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT), + .alpha_support = IS_ENABLED(CONFIG_DRM_I915_ALPHA_SUPPORT), .disable_power_well = -1, .enable_ips = 1, .fastboot = 0, @@ -145,9 +145,10 @@ MODULE_PARM_DESC(enable_psr, "Enable PSR " "(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) " "Default: -1 (use per-chip default)"); -module_param_named_unsafe(preliminary_hw_support, i915.preliminary_hw_support, int, 0400); -MODULE_PARM_DESC(preliminary_hw_support, - "Enable preliminary hardware support."); +module_param_named_unsafe(alpha_support, i915.alpha_support, int, 0400); +MODULE_PARM_DESC(alpha_support, + "Enable alpha quality driver support for latest hardware. " + "See also CONFIG_DRM_I915_ALPHA_SUPPORT."); module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0400); MODULE_PARM_DESC(disable_power_well, diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index 94efc899c1ef..817ad959941e 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -40,7 +40,7 @@ struct i915_params { int enable_ppgtt; int enable_execlists; int enable_psr; - unsigned int preliminary_hw_support; + unsigned int alpha_support; int disable_power_well; int enable_ips; int invert_brightness; diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 31e6edd08dd0..fce8e198bc76 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -288,7 +288,8 @@ static const struct intel_device_info intel_haswell_info = { #define BDW_FEATURES \ HSW_FEATURES, \ BDW_COLORS, \ - .has_logical_ring_contexts = 1 + .has_logical_ring_contexts = 1, \ + .has_64bit_reloc = 1 static const struct intel_device_info intel_broadwell_info = { BDW_FEATURES, @@ -308,6 +309,7 @@ static const struct intel_device_info intel_cherryview_info = { .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .is_cherryview = 1, + .has_64bit_reloc = 1, .has_psr = 1, .has_runtime_pm = 1, .has_resource_streamer = 1, @@ -347,6 +349,7 @@ static const struct intel_device_info intel_broxton_info = { .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .num_pipes = 3, + .has_64bit_reloc = 1, .has_ddi = 1, .has_fpga_dbg = 1, .has_fbc = 1, @@ -360,6 +363,7 @@ static const struct intel_device_info intel_broxton_info = { .has_hw_contexts = 1, .has_logical_ring_contexts = 1, .has_guc = 1, + .has_decoupled_mmio = 1, .ddb_size = 512, GEN_DEFAULT_PIPEOFFSETS, IVB_CURSOR_OFFSETS, @@ -436,9 +440,10 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct intel_device_info *intel_info = (struct intel_device_info *) ent->driver_data; - if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) { - DRM_INFO("This hardware requires preliminary hardware support.\n" - "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n"); + if (IS_ALPHA_SUPPORT(intel_info) && !i915.alpha_support) { + DRM_INFO("The driver support for your hardware in this kernel version is alpha quality\n" + "See CONFIG_DRM_I915_ALPHA_SUPPORT or i915.alpha_support module parameter\n" + "to enable support in this kernel version, or check for kernel updates.\n"); return -ENODEV; } diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 00efaa13974d..c70c07a7b586 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -830,96 +830,7 @@ enum skl_disp_power_wells { #define CCK_FREQUENCY_STATUS_SHIFT 8 #define CCK_FREQUENCY_VALUES (0x1f << 0) -/** - * DOC: DPIO - * - * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI - * ports. DPIO is the name given to such a display PHY. These PHYs - * don't follow the standard programming model using direct MMIO - * registers, and instead their registers must be accessed trough IOSF - * sideband. VLV has one such PHY for driving ports B and C, and CHV - * adds another PHY for driving port D. Each PHY responds to specific - * IOSF-SB port. - * - * Each display PHY is made up of one or two channels. Each channel - * houses a common lane part which contains the PLL and other common - * logic. CH0 common lane also contains the IOSF-SB logic for the - * Common Register Interface (CRI) ie. the DPIO registers. CRI clock - * must be running when any DPIO registers are accessed. - * - * In addition to having their own registers, the PHYs are also - * controlled through some dedicated signals from the display - * controller. These include PLL reference clock enable, PLL enable, - * and CRI clock selection, for example. - * - * Eeach channel also has two splines (also called data lanes), and - * each spline is made up of one Physical Access Coding Sub-Layer - * (PCS) block and two TX lanes. So each channel has two PCS blocks - * and four TX lanes. The TX lanes are used as DP lanes or TMDS - * data/clock pairs depending on the output type. - * - * Additionally the PHY also contains an AUX lane with AUX blocks - * for each channel. This is used for DP AUX communication, but - * this fact isn't really relevant for the driver since AUX is - * controlled from the display controller side. No DPIO registers - * need to be accessed during AUX communication, - * - * Generally on VLV/CHV the common lane corresponds to the pipe and - * the spline (PCS/TX) corresponds to the port. - * - * For dual channel PHY (VLV/CHV): - * - * pipe A == CMN/PLL/REF CH0 - * - * pipe B == CMN/PLL/REF CH1 - * - * port B == PCS/TX CH0 - * - * port C == PCS/TX CH1 - * - * This is especially important when we cross the streams - * ie. drive port B with pipe B, or port C with pipe A. - * - * For single channel PHY (CHV): - * - * pipe C == CMN/PLL/REF CH0 - * - * port D == PCS/TX CH0 - * - * On BXT the entire PHY channel corresponds to the port. That means - * the PLL is also now associated with the port rather than the pipe, - * and so the clock needs to be routed to the appropriate transcoder. - * Port A PLL is directly connected to transcoder EDP and port B/C - * PLLs can be routed to any transcoder A/B/C. - * - * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is - * digital port D (CHV) or port A (BXT). :: - * - * - * Dual channel PHY (VLV/CHV/BXT) - * --------------------------------- - * | CH0 | CH1 | - * | CMN/PLL/REF | CMN/PLL/REF | - * |---------------|---------------| Display PHY - * | PCS01 | PCS23 | PCS01 | PCS23 | - * |-------|-------|-------|-------| - * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| - * --------------------------------- - * | DDI0 | DDI1 | DP/HDMI ports - * --------------------------------- - * - * Single channel PHY (CHV/BXT) - * ----------------- - * | CH0 | - * | CMN/PLL/REF | - * |---------------| Display PHY - * | PCS01 | PCS23 | - * |-------|-------| - * |TX0|TX1|TX2|TX3| - * ----------------- - * | DDI2 | DP/HDMI port - * ----------------- - */ +/* DPIO registers */ #define DPIO_DEVFN 0 #define DPIO_CTL _MMIO(VLV_DISPLAY_BASE + 0x2110) @@ -1275,7 +1186,19 @@ enum skl_disp_power_wells { #define DPIO_UPAR_SHIFT 30 /* BXT PHY registers */ -#define _BXT_PHY(phy, a, b) _MMIO_PIPE((phy), (a), (b)) +#define _BXT_PHY0_BASE 0x6C000 +#define _BXT_PHY1_BASE 0x162000 +#define BXT_PHY_BASE(phy) _PIPE((phy), _BXT_PHY0_BASE, \ + _BXT_PHY1_BASE) + +#define _BXT_PHY(phy, reg) \ + _MMIO(BXT_PHY_BASE(phy) - _BXT_PHY0_BASE + (reg)) + +#define _BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \ + (BXT_PHY_BASE(phy) + _PIPE((ch), (reg_ch0) - _BXT_PHY0_BASE, \ + (reg_ch1) - _BXT_PHY0_BASE)) +#define _MMIO_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \ + _MMIO(_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1)) #define BXT_P_CR_GT_DISP_PWRON _MMIO(0x138090) #define GT_DISPLAY_POWER_ON(phy) (1 << (phy)) @@ -1292,8 +1215,8 @@ enum skl_disp_power_wells { #define _PHY_CTL_FAMILY_EDP 0x64C80 #define _PHY_CTL_FAMILY_DDI 0x64C90 #define COMMON_RESET_DIS (1 << 31) -#define BXT_PHY_CTL_FAMILY(phy) _BXT_PHY((phy), _PHY_CTL_FAMILY_DDI, \ - _PHY_CTL_FAMILY_EDP) +#define BXT_PHY_CTL_FAMILY(phy) _MMIO_PIPE((phy), _PHY_CTL_FAMILY_DDI, \ + _PHY_CTL_FAMILY_EDP) /* BXT PHY PLL registers */ #define _PORT_PLL_A 0x46074 @@ -1313,18 +1236,18 @@ enum skl_disp_power_wells { #define PORT_PLL_P2_SHIFT 8 #define PORT_PLL_P2_MASK (0x1f << PORT_PLL_P2_SHIFT) #define PORT_PLL_P2(x) ((x) << PORT_PLL_P2_SHIFT) -#define BXT_PORT_PLL_EBB_0(port) _MMIO_PORT3(port, _PORT_PLL_EBB_0_A, \ - _PORT_PLL_EBB_0_B, \ - _PORT_PLL_EBB_0_C) +#define BXT_PORT_PLL_EBB_0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PLL_EBB_0_B, \ + _PORT_PLL_EBB_0_C) #define _PORT_PLL_EBB_4_A 0x162038 #define _PORT_PLL_EBB_4_B 0x6C038 #define _PORT_PLL_EBB_4_C 0x6C344 #define PORT_PLL_10BIT_CLK_ENABLE (1 << 13) #define PORT_PLL_RECALIBRATE (1 << 14) -#define BXT_PORT_PLL_EBB_4(port) _MMIO_PORT3(port, _PORT_PLL_EBB_4_A, \ - _PORT_PLL_EBB_4_B, \ - _PORT_PLL_EBB_4_C) +#define BXT_PORT_PLL_EBB_4(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PLL_EBB_4_B, \ + _PORT_PLL_EBB_4_C) #define _PORT_PLL_0_A 0x162100 #define _PORT_PLL_0_B 0x6C100 @@ -1355,57 +1278,56 @@ enum skl_disp_power_wells { #define PORT_PLL_DCO_AMP_DEFAULT 15 #define PORT_PLL_DCO_AMP_MASK 0x3c00 #define PORT_PLL_DCO_AMP(x) ((x)<<10) -#define _PORT_PLL_BASE(port) _PORT3(port, _PORT_PLL_0_A, \ - _PORT_PLL_0_B, \ - _PORT_PLL_0_C) -#define BXT_PORT_PLL(port, idx) _MMIO(_PORT_PLL_BASE(port) + (idx) * 4) +#define _PORT_PLL_BASE(phy, ch) _BXT_PHY_CH(phy, ch, \ + _PORT_PLL_0_B, \ + _PORT_PLL_0_C) +#define BXT_PORT_PLL(phy, ch, idx) _MMIO(_PORT_PLL_BASE(phy, ch) + \ + (idx) * 4) /* BXT PHY common lane registers */ #define _PORT_CL1CM_DW0_A 0x162000 #define _PORT_CL1CM_DW0_BC 0x6C000 #define PHY_POWER_GOOD (1 << 16) #define PHY_RESERVED (1 << 7) -#define BXT_PORT_CL1CM_DW0(phy) _BXT_PHY((phy), _PORT_CL1CM_DW0_BC, \ - _PORT_CL1CM_DW0_A) +#define BXT_PORT_CL1CM_DW0(phy) _BXT_PHY((phy), _PORT_CL1CM_DW0_BC) #define _PORT_CL1CM_DW9_A 0x162024 #define _PORT_CL1CM_DW9_BC 0x6C024 #define IREF0RC_OFFSET_SHIFT 8 #define IREF0RC_OFFSET_MASK (0xFF << IREF0RC_OFFSET_SHIFT) -#define BXT_PORT_CL1CM_DW9(phy) _BXT_PHY((phy), _PORT_CL1CM_DW9_BC, \ - _PORT_CL1CM_DW9_A) +#define BXT_PORT_CL1CM_DW9(phy) _BXT_PHY((phy), _PORT_CL1CM_DW9_BC) #define _PORT_CL1CM_DW10_A 0x162028 #define _PORT_CL1CM_DW10_BC 0x6C028 #define IREF1RC_OFFSET_SHIFT 8 #define IREF1RC_OFFSET_MASK (0xFF << IREF1RC_OFFSET_SHIFT) -#define BXT_PORT_CL1CM_DW10(phy) _BXT_PHY((phy), _PORT_CL1CM_DW10_BC, \ - _PORT_CL1CM_DW10_A) +#define BXT_PORT_CL1CM_DW10(phy) _BXT_PHY((phy), _PORT_CL1CM_DW10_BC) #define _PORT_CL1CM_DW28_A 0x162070 #define _PORT_CL1CM_DW28_BC 0x6C070 #define OCL1_POWER_DOWN_EN (1 << 23) #define DW28_OLDO_DYN_PWR_DOWN_EN (1 << 22) #define SUS_CLK_CONFIG 0x3 -#define BXT_PORT_CL1CM_DW28(phy) _BXT_PHY((phy), _PORT_CL1CM_DW28_BC, \ - _PORT_CL1CM_DW28_A) +#define BXT_PORT_CL1CM_DW28(phy) _BXT_PHY((phy), _PORT_CL1CM_DW28_BC) #define _PORT_CL1CM_DW30_A 0x162078 #define _PORT_CL1CM_DW30_BC 0x6C078 #define OCL2_LDOFUSE_PWR_DIS (1 << 6) -#define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC, \ - _PORT_CL1CM_DW30_A) +#define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC) -/* Defined for PHY0 only */ -#define BXT_PORT_CL2CM_DW6_BC _MMIO(0x6C358) +/* The spec defines this only for BXT PHY0, but lets assume that this + * would exist for PHY1 too if it had a second channel. + */ +#define _PORT_CL2CM_DW6_A 0x162358 +#define _PORT_CL2CM_DW6_BC 0x6C358 +#define BXT_PORT_CL2CM_DW6(phy) _BXT_PHY((phy), _PORT_CL2CM_DW6_BC) #define DW6_OLDO_DYN_PWR_DOWN_EN (1 << 28) /* BXT PHY Ref registers */ #define _PORT_REF_DW3_A 0x16218C #define _PORT_REF_DW3_BC 0x6C18C #define GRC_DONE (1 << 22) -#define BXT_PORT_REF_DW3(phy) _BXT_PHY((phy), _PORT_REF_DW3_BC, \ - _PORT_REF_DW3_A) +#define BXT_PORT_REF_DW3(phy) _BXT_PHY((phy), _PORT_REF_DW3_BC) #define _PORT_REF_DW6_A 0x162198 #define _PORT_REF_DW6_BC 0x6C198 @@ -1416,15 +1338,13 @@ enum skl_disp_power_wells { #define GRC_CODE_SLOW_SHIFT 8 #define GRC_CODE_SLOW_MASK (0xFF << GRC_CODE_SLOW_SHIFT) #define GRC_CODE_NOM_MASK 0xFF -#define BXT_PORT_REF_DW6(phy) _BXT_PHY((phy), _PORT_REF_DW6_BC, \ - _PORT_REF_DW6_A) +#define BXT_PORT_REF_DW6(phy) _BXT_PHY((phy), _PORT_REF_DW6_BC) #define _PORT_REF_DW8_A 0x1621A0 #define _PORT_REF_DW8_BC 0x6C1A0 #define GRC_DIS (1 << 15) #define GRC_RDY_OVRD (1 << 1) -#define BXT_PORT_REF_DW8(phy) _BXT_PHY((phy), _PORT_REF_DW8_BC, \ - _PORT_REF_DW8_A) +#define BXT_PORT_REF_DW8(phy) _BXT_PHY((phy), _PORT_REF_DW8_BC) /* BXT PHY PCS registers */ #define _PORT_PCS_DW10_LN01_A 0x162428 @@ -1433,12 +1353,13 @@ enum skl_disp_power_wells { #define _PORT_PCS_DW10_GRP_A 0x162C28 #define _PORT_PCS_DW10_GRP_B 0x6CC28 #define _PORT_PCS_DW10_GRP_C 0x6CE28 -#define BXT_PORT_PCS_DW10_LN01(port) _MMIO_PORT3(port, _PORT_PCS_DW10_LN01_A, \ - _PORT_PCS_DW10_LN01_B, \ - _PORT_PCS_DW10_LN01_C) -#define BXT_PORT_PCS_DW10_GRP(port) _MMIO_PORT3(port, _PORT_PCS_DW10_GRP_A, \ - _PORT_PCS_DW10_GRP_B, \ - _PORT_PCS_DW10_GRP_C) +#define BXT_PORT_PCS_DW10_LN01(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW10_LN01_B, \ + _PORT_PCS_DW10_LN01_C) +#define BXT_PORT_PCS_DW10_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW10_GRP_B, \ + _PORT_PCS_DW10_GRP_C) + #define TX2_SWING_CALC_INIT (1 << 31) #define TX1_SWING_CALC_INIT (1 << 30) @@ -1453,15 +1374,15 @@ enum skl_disp_power_wells { #define _PORT_PCS_DW12_GRP_C 0x6CE30 #define LANESTAGGER_STRAP_OVRD (1 << 6) #define LANE_STAGGER_MASK 0x1F -#define BXT_PORT_PCS_DW12_LN01(port) _MMIO_PORT3(port, _PORT_PCS_DW12_LN01_A, \ - _PORT_PCS_DW12_LN01_B, \ - _PORT_PCS_DW12_LN01_C) -#define BXT_PORT_PCS_DW12_LN23(port) _MMIO_PORT3(port, _PORT_PCS_DW12_LN23_A, \ - _PORT_PCS_DW12_LN23_B, \ - _PORT_PCS_DW12_LN23_C) -#define BXT_PORT_PCS_DW12_GRP(port) _MMIO_PORT3(port, _PORT_PCS_DW12_GRP_A, \ - _PORT_PCS_DW12_GRP_B, \ - _PORT_PCS_DW12_GRP_C) +#define BXT_PORT_PCS_DW12_LN01(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW12_LN01_B, \ + _PORT_PCS_DW12_LN01_C) +#define BXT_PORT_PCS_DW12_LN23(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW12_LN23_B, \ + _PORT_PCS_DW12_LN23_C) +#define BXT_PORT_PCS_DW12_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW12_GRP_B, \ + _PORT_PCS_DW12_GRP_C) /* BXT PHY TX registers */ #define _BXT_LANE_OFFSET(lane) (((lane) >> 1) * 0x200 + \ @@ -1473,12 +1394,12 @@ enum skl_disp_power_wells { #define _PORT_TX_DW2_GRP_A 0x162D08 #define _PORT_TX_DW2_GRP_B 0x6CD08 #define _PORT_TX_DW2_GRP_C 0x6CF08 -#define BXT_PORT_TX_DW2_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW2_GRP_A, \ - _PORT_TX_DW2_GRP_B, \ - _PORT_TX_DW2_GRP_C) -#define BXT_PORT_TX_DW2_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW2_LN0_A, \ - _PORT_TX_DW2_LN0_B, \ - _PORT_TX_DW2_LN0_C) +#define BXT_PORT_TX_DW2_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW2_LN0_B, \ + _PORT_TX_DW2_LN0_C) +#define BXT_PORT_TX_DW2_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW2_GRP_B, \ + _PORT_TX_DW2_GRP_C) #define MARGIN_000_SHIFT 16 #define MARGIN_000 (0xFF << MARGIN_000_SHIFT) #define UNIQ_TRANS_SCALE_SHIFT 8 @@ -1490,12 +1411,12 @@ enum skl_disp_power_wells { #define _PORT_TX_DW3_GRP_A 0x162D0C #define _PORT_TX_DW3_GRP_B 0x6CD0C #define _PORT_TX_DW3_GRP_C 0x6CF0C -#define BXT_PORT_TX_DW3_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW3_GRP_A, \ - _PORT_TX_DW3_GRP_B, \ - _PORT_TX_DW3_GRP_C) -#define BXT_PORT_TX_DW3_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW3_LN0_A, \ - _PORT_TX_DW3_LN0_B, \ - _PORT_TX_DW3_LN0_C) +#define BXT_PORT_TX_DW3_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW3_LN0_B, \ + _PORT_TX_DW3_LN0_C) +#define BXT_PORT_TX_DW3_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW3_GRP_B, \ + _PORT_TX_DW3_GRP_C) #define SCALE_DCOMP_METHOD (1 << 26) #define UNIQUE_TRANGE_EN_METHOD (1 << 27) @@ -1505,12 +1426,12 @@ enum skl_disp_power_wells { #define _PORT_TX_DW4_GRP_A 0x162D10 #define _PORT_TX_DW4_GRP_B 0x6CD10 #define _PORT_TX_DW4_GRP_C 0x6CF10 -#define BXT_PORT_TX_DW4_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW4_LN0_A, \ - _PORT_TX_DW4_LN0_B, \ - _PORT_TX_DW4_LN0_C) -#define BXT_PORT_TX_DW4_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW4_GRP_A, \ - _PORT_TX_DW4_GRP_B, \ - _PORT_TX_DW4_GRP_C) +#define BXT_PORT_TX_DW4_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW4_LN0_B, \ + _PORT_TX_DW4_LN0_C) +#define BXT_PORT_TX_DW4_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW4_GRP_B, \ + _PORT_TX_DW4_GRP_C) #define DEEMPH_SHIFT 24 #define DE_EMPHASIS (0xFF << DEEMPH_SHIFT) @@ -1519,10 +1440,10 @@ enum skl_disp_power_wells { #define _PORT_TX_DW14_LN0_C 0x6C938 #define LATENCY_OPTIM_SHIFT 30 #define LATENCY_OPTIM (1 << LATENCY_OPTIM_SHIFT) -#define BXT_PORT_TX_DW14_LN(port, lane) _MMIO(_PORT3((port), _PORT_TX_DW14_LN0_A, \ - _PORT_TX_DW14_LN0_B, \ - _PORT_TX_DW14_LN0_C) + \ - _BXT_LANE_OFFSET(lane)) +#define BXT_PORT_TX_DW14_LN(phy, ch, lane) \ + _MMIO(_BXT_PHY_CH(phy, ch, _PORT_TX_DW14_LN0_B, \ + _PORT_TX_DW14_LN0_C) + \ + _BXT_LANE_OFFSET(lane)) /* UAIMI scratch pad register 1 */ #define UAIMI_SPR1 _MMIO(0x4F074) @@ -2188,8 +2109,9 @@ enum skl_disp_power_wells { #define FBC_FENCE_OFF _MMIO(0x3218) /* BSpec typo has 321Bh */ #define FBC_TAG(i) _MMIO(0x3300 + (i) * 4) -#define FBC_STATUS2 _MMIO(0x43214) -#define FBC_COMPRESSION_MASK 0x7ff +#define FBC_STATUS2 _MMIO(0x43214) +#define IVB_FBC_COMPRESSION_MASK 0x7ff +#define BDW_FBC_COMPRESSION_MASK 0xfff #define FBC_LL_SIZE (1536) @@ -6015,6 +5937,7 @@ enum { #define GEN8_DE_PIPE_A_IRQ (1<<16) #define GEN8_DE_PIPE_IRQ(pipe) (1<<(16+(pipe))) #define GEN8_GT_VECS_IRQ (1<<6) +#define GEN8_GT_GUC_IRQ (1<<5) #define GEN8_GT_PM_IRQ (1<<4) #define GEN8_GT_VCS2_IRQ (1<<3) #define GEN8_GT_VCS1_IRQ (1<<2) @@ -6026,6 +5949,16 @@ enum { #define GEN8_GT_IIR(which) _MMIO(0x44308 + (0x10 * (which))) #define GEN8_GT_IER(which) _MMIO(0x4430c + (0x10 * (which))) +#define GEN9_GUC_TO_HOST_INT_EVENT (1<<31) +#define GEN9_GUC_EXEC_ERROR_EVENT (1<<30) +#define GEN9_GUC_DISPLAY_EVENT (1<<29) +#define GEN9_GUC_SEMA_SIGNAL_EVENT (1<<28) +#define GEN9_GUC_IOMMU_MSG_EVENT (1<<27) +#define GEN9_GUC_DB_RING_EVENT (1<<26) +#define GEN9_GUC_DMA_DONE_EVENT (1<<25) +#define GEN9_GUC_FATAL_ERROR_EVENT (1<<24) +#define GEN9_GUC_NOTIFICATION_EVENT (1<<23) + #define GEN8_RCS_IRQ_SHIFT 0 #define GEN8_BCS_IRQ_SHIFT 16 #define GEN8_VCS1_IRQ_SHIFT 0 @@ -7358,6 +7291,13 @@ enum { #define _HSW_AUD_MISC_CTRL_B 0x65110 #define HSW_AUD_MISC_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_MISC_CTRL_A, _HSW_AUD_MISC_CTRL_B) +#define _HSW_AUD_M_CTS_ENABLE_A 0x65028 +#define _HSW_AUD_M_CTS_ENABLE_B 0x65128 +#define HSW_AUD_M_CTS_ENABLE(pipe) _MMIO_PIPE(pipe, _HSW_AUD_M_CTS_ENABLE_A, _HSW_AUD_M_CTS_ENABLE_B) +#define AUD_M_CTS_M_VALUE_INDEX (1 << 21) +#define AUD_M_CTS_M_PROG_ENABLE (1 << 20) +#define AUD_CONFIG_M_MASK 0xfffff + #define _HSW_AUD_DIP_ELD_CTRL_ST_A 0x650b4 #define _HSW_AUD_DIP_ELD_CTRL_ST_B 0x651b4 #define HSW_AUD_DIP_ELD_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_DIP_ELD_CTRL_ST_A, _HSW_AUD_DIP_ELD_CTRL_ST_B) @@ -7402,6 +7342,13 @@ enum { #define SKL_FUSE_PG1_DIST_STATUS (1<<26) #define SKL_FUSE_PG2_DIST_STATUS (1<<25) +/* Decoupled MMIO register pair for kernel driver */ +#define GEN9_DECOUPLED_REG0_DW0 _MMIO(0xF00) +#define GEN9_DECOUPLED_REG0_DW1 _MMIO(0xF04) +#define GEN9_DECOUPLED_DW1_GO (1<<31) +#define GEN9_DECOUPLED_PD_SHIFT 28 +#define GEN9_DECOUPLED_OP_SHIFT 24 + /* Per-pipe DDI Function Control */ #define _TRANS_DDI_FUNC_CTL_A 0x60400 #define _TRANS_DDI_FUNC_CTL_B 0x61400 diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index 344cbf39cfa9..b0e1e7ca75da 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -29,12 +29,10 @@ #include "intel_drv.h" #include "i915_reg.h" -static void i915_save_display(struct drm_device *dev) +static void i915_save_display(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* Display arbitration control */ - if (INTEL_INFO(dev)->gen <= 4) + if (INTEL_GEN(dev_priv) <= 4) dev_priv->regfile.saveDSPARB = I915_READ(DSPARB); /* save FBC interval */ @@ -42,12 +40,10 @@ static void i915_save_display(struct drm_device *dev) dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL); } -static void i915_restore_display(struct drm_device *dev) +static void i915_restore_display(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* Display arbitration */ - if (INTEL_INFO(dev)->gen <= 4) + if (INTEL_GEN(dev_priv) <= 4) I915_WRITE(DSPARB, dev_priv->regfile.saveDSPARB); /* only restore FBC info on the platform that supports FBC*/ @@ -57,7 +53,7 @@ static void i915_restore_display(struct drm_device *dev) if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv)) I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL); - i915_redisable_vga(dev); + i915_redisable_vga(dev_priv); } int i915_save_state(struct drm_device *dev) @@ -68,14 +64,14 @@ int i915_save_state(struct drm_device *dev) mutex_lock(&dev->struct_mutex); - i915_save_display(dev); + i915_save_display(dev_priv); if (IS_GEN4(dev_priv)) pci_read_config_word(pdev, GCDGMBUS, &dev_priv->regfile.saveGCDGMBUS); /* Cache mode state */ - if (INTEL_INFO(dev)->gen < 7) + if (INTEL_GEN(dev_priv) < 7) dev_priv->regfile.saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0); /* Memory Arbitration state */ @@ -114,15 +110,15 @@ int i915_restore_state(struct drm_device *dev) mutex_lock(&dev->struct_mutex); - i915_gem_restore_fences(dev); + i915_gem_restore_fences(dev_priv); if (IS_GEN4(dev_priv)) pci_write_config_word(pdev, GCDGMBUS, dev_priv->regfile.saveGCDGMBUS); - i915_restore_display(dev); + i915_restore_display(dev_priv); /* Cache mode state */ - if (INTEL_INFO(dev)->gen < 7) + if (INTEL_GEN(dev_priv) < 7) I915_WRITE(CACHE_MODE_0, dev_priv->regfile.saveCACHE_MODE_0 | 0xffff0000); diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c index 8185002d7ec8..147420ccf49c 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.c +++ b/drivers/gpu/drm/i915/i915_sw_fence.c @@ -13,6 +13,8 @@ #include "i915_sw_fence.h" +#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ + static DEFINE_SPINLOCK(i915_sw_fence_lock); static int __i915_sw_fence_notify(struct i915_sw_fence *fence, @@ -114,11 +116,14 @@ static void i915_sw_fence_await(struct i915_sw_fence *fence) WARN_ON(atomic_inc_return(&fence->pending) <= 1); } -void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn) +void __i915_sw_fence_init(struct i915_sw_fence *fence, + i915_sw_fence_notify_t fn, + const char *name, + struct lock_class_key *key) { BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK); - init_waitqueue_head(&fence->wait); + __init_waitqueue_head(&fence->wait, name, key); kref_init(&fence->kref); atomic_set(&fence->pending, 1); fence->flags = (unsigned long)fn; @@ -135,6 +140,8 @@ static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void * list_del(&wq->task_list); __i915_sw_fence_complete(wq->private, key); i915_sw_fence_put(wq->private); + if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) + kfree(wq); return 0; } @@ -192,9 +199,9 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, return err; } -int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, - struct i915_sw_fence *signaler, - wait_queue_t *wq) +static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, + struct i915_sw_fence *signaler, + wait_queue_t *wq, gfp_t gfp) { unsigned long flags; int pending; @@ -206,8 +213,22 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) return -EINVAL; + pending = 0; + if (!wq) { + wq = kmalloc(sizeof(*wq), gfp); + if (!wq) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + i915_sw_fence_wait(signaler); + return 0; + } + + pending |= I915_SW_FENCE_FLAG_ALLOC; + } + INIT_LIST_HEAD(&wq->task_list); - wq->flags = 0; + wq->flags = pending; wq->func = i915_sw_fence_wake; wq->private = i915_sw_fence_get(fence); @@ -226,6 +247,20 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, return pending; } +int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, + struct i915_sw_fence *signaler, + wait_queue_t *wq) +{ + return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); +} + +int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, + struct i915_sw_fence *signaler, + gfp_t gfp) +{ + return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); +} + struct i915_sw_dma_fence_cb { struct dma_fence_cb base; struct i915_sw_fence *fence; diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h index cd239e92f67f..0f3185ef7f4e 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.h +++ b/drivers/gpu/drm/i915/i915_sw_fence.h @@ -40,12 +40,30 @@ typedef int (*i915_sw_fence_notify_t)(struct i915_sw_fence *, enum i915_sw_fence_notify state); #define __i915_sw_fence_call __aligned(4) -void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn); +void __i915_sw_fence_init(struct i915_sw_fence *fence, + i915_sw_fence_notify_t fn, + const char *name, + struct lock_class_key *key); +#ifdef CONFIG_LOCKDEP +#define i915_sw_fence_init(fence, fn) \ +do { \ + static struct lock_class_key __key; \ + \ + __i915_sw_fence_init((fence), (fn), #fence, &__key); \ +} while (0) +#else +#define i915_sw_fence_init(fence, fn) \ + __i915_sw_fence_init((fence), (fn), NULL, NULL) +#endif + void i915_sw_fence_commit(struct i915_sw_fence *fence); int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, struct i915_sw_fence *after, wait_queue_t *wq); +int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, + struct i915_sw_fence *after, + gfp_t gfp); int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, struct dma_fence *dma, unsigned long timeout, @@ -57,9 +75,19 @@ int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, unsigned long timeout, gfp_t gfp); +static inline bool i915_sw_fence_signaled(const struct i915_sw_fence *fence) +{ + return atomic_read(&fence->pending) <= 0; +} + static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence) { return atomic_read(&fence->pending) < 0; } +static inline void i915_sw_fence_wait(struct i915_sw_fence *fence) +{ + wait_event(fence->wait, i915_sw_fence_done(fence)); +} + #endif /* _I915_SW_FENCE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 5c912c25f7d3..c5d210ebaa9a 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -466,7 +466,7 @@ TRACE_EVENT(i915_gem_ring_sync_to, __entry->dev = from->i915->drm.primary->index; __entry->sync_from = from->engine->id; __entry->sync_to = to->engine->id; - __entry->seqno = from->fence.seqno; + __entry->seqno = from->global_seqno; ), TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u", @@ -489,7 +489,7 @@ TRACE_EVENT(i915_gem_ring_dispatch, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->flags = flags; dma_fence_enable_sw_signaling(&req->fence); ), @@ -534,7 +534,7 @@ DECLARE_EVENT_CLASS(i915_gem_request, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; ), TP_printk("dev=%u, ring=%u, seqno=%u", @@ -596,7 +596,7 @@ TRACE_EVENT(i915_gem_request_wait_begin, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->blocking = mutex_is_locked(&req->i915->drm.struct_mutex); ), diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c new file mode 100644 index 000000000000..a792dcb902b5 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -0,0 +1,638 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "i915_vma.h" + +#include "i915_drv.h" +#include "intel_ringbuffer.h" +#include "intel_frontbuffer.h" + +#include <drm/drm_gem.h> + +static void +i915_vma_retire(struct i915_gem_active *active, + struct drm_i915_gem_request *rq) +{ + const unsigned int idx = rq->engine->id; + struct i915_vma *vma = + container_of(active, struct i915_vma, last_read[idx]); + struct drm_i915_gem_object *obj = vma->obj; + + GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); + + i915_vma_clear_active(vma, idx); + if (i915_vma_is_active(vma)) + return; + + list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma))) + WARN_ON(i915_vma_unbind(vma)); + + GEM_BUG_ON(!i915_gem_object_is_active(obj)); + if (--obj->active_count) + return; + + /* Bump our place on the bound list to keep it roughly in LRU order + * so that we don't steal from recently used but inactive objects + * (unless we are forced to ofc!) + */ + if (obj->bind_count) + list_move_tail(&obj->global_link, &rq->i915->mm.bound_list); + + obj->mm.dirty = true; /* be paranoid */ + + if (i915_gem_object_has_active_reference(obj)) { + i915_gem_object_clear_active_reference(obj); + i915_gem_object_put(obj); + } +} + +static struct i915_vma * +__i915_vma_create(struct drm_i915_gem_object *obj, + struct i915_address_space *vm, + const struct i915_ggtt_view *view) +{ + struct i915_vma *vma; + struct rb_node *rb, **p; + int i; + + GEM_BUG_ON(vm->closed); + + vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); + if (vma == NULL) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&vma->exec_list); + for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) + init_request_active(&vma->last_read[i], i915_vma_retire); + init_request_active(&vma->last_fence, NULL); + list_add(&vma->vm_link, &vm->unbound_list); + vma->vm = vm; + vma->obj = obj; + vma->size = obj->base.size; + + if (view) { + vma->ggtt_view = *view; + if (view->type == I915_GGTT_VIEW_PARTIAL) { + vma->size = view->params.partial.size; + vma->size <<= PAGE_SHIFT; + } else if (view->type == I915_GGTT_VIEW_ROTATED) { + vma->size = + intel_rotation_info_size(&view->params.rotated); + vma->size <<= PAGE_SHIFT; + } + } + + if (i915_is_ggtt(vm)) { + vma->flags |= I915_VMA_GGTT; + list_add(&vma->obj_link, &obj->vma_list); + } else { + i915_ppgtt_get(i915_vm_to_ppgtt(vm)); + list_add_tail(&vma->obj_link, &obj->vma_list); + } + + rb = NULL; + p = &obj->vma_tree.rb_node; + while (*p) { + struct i915_vma *pos; + + rb = *p; + pos = rb_entry(rb, struct i915_vma, obj_node); + if (i915_vma_compare(pos, vm, view) < 0) + p = &rb->rb_right; + else + p = &rb->rb_left; + } + rb_link_node(&vma->obj_node, rb, p); + rb_insert_color(&vma->obj_node, &obj->vma_tree); + + return vma; +} + +struct i915_vma * +i915_vma_create(struct drm_i915_gem_object *obj, + struct i915_address_space *vm, + const struct i915_ggtt_view *view) +{ + lockdep_assert_held(&obj->base.dev->struct_mutex); + GEM_BUG_ON(view && !i915_is_ggtt(vm)); + GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view)); + + return __i915_vma_create(obj, vm, view); +} + +/** + * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. + * @vma: VMA to map + * @cache_level: mapping cache level + * @flags: flags like global or local mapping + * + * DMA addresses are taken from the scatter-gather table of this object (or of + * this VMA in case of non-default GGTT views) and PTE entries set up. + * Note that DMA addresses are also the only part of the SG table we care about. + */ +int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, + u32 flags) +{ + u32 bind_flags; + u32 vma_flags; + int ret; + + if (WARN_ON(flags == 0)) + return -EINVAL; + + bind_flags = 0; + if (flags & PIN_GLOBAL) + bind_flags |= I915_VMA_GLOBAL_BIND; + if (flags & PIN_USER) + bind_flags |= I915_VMA_LOCAL_BIND; + + vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); + if (flags & PIN_UPDATE) + bind_flags |= vma_flags; + else + bind_flags &= ~vma_flags; + if (bind_flags == 0) + return 0; + + if (vma_flags == 0 && vma->vm->allocate_va_range) { + trace_i915_va_alloc(vma); + ret = vma->vm->allocate_va_range(vma->vm, + vma->node.start, + vma->node.size); + if (ret) + return ret; + } + + ret = vma->vm->bind_vma(vma, cache_level, bind_flags); + if (ret) + return ret; + + vma->flags |= bind_flags; + return 0; +} + +void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) +{ + void __iomem *ptr; + + /* Access through the GTT requires the device to be awake. */ + assert_rpm_wakelock_held(to_i915(vma->vm->dev)); + + lockdep_assert_held(&vma->vm->dev->struct_mutex); + if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) + return IO_ERR_PTR(-ENODEV); + + GEM_BUG_ON(!i915_vma_is_ggtt(vma)); + GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0); + + ptr = vma->iomap; + if (ptr == NULL) { + ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable, + vma->node.start, + vma->node.size); + if (ptr == NULL) + return IO_ERR_PTR(-ENOMEM); + + vma->iomap = ptr; + } + + __i915_vma_pin(vma); + return ptr; +} + +void i915_vma_unpin_and_release(struct i915_vma **p_vma) +{ + struct i915_vma *vma; + struct drm_i915_gem_object *obj; + + vma = fetch_and_zero(p_vma); + if (!vma) + return; + + obj = vma->obj; + + i915_vma_unpin(vma); + i915_vma_close(vma); + + __i915_gem_object_release_unless_active(obj); +} + +bool +i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) +{ + if (!drm_mm_node_allocated(&vma->node)) + return false; + + if (vma->node.size < size) + return true; + + if (alignment && vma->node.start & (alignment - 1)) + return true; + + if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) + return true; + + if (flags & PIN_OFFSET_BIAS && + vma->node.start < (flags & PIN_OFFSET_MASK)) + return true; + + if (flags & PIN_OFFSET_FIXED && + vma->node.start != (flags & PIN_OFFSET_MASK)) + return true; + + return false; +} + +void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) +{ + struct drm_i915_gem_object *obj = vma->obj; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); + bool mappable, fenceable; + u32 fence_size, fence_alignment; + + fence_size = i915_gem_get_ggtt_size(dev_priv, + vma->size, + i915_gem_object_get_tiling(obj)); + fence_alignment = i915_gem_get_ggtt_alignment(dev_priv, + vma->size, + i915_gem_object_get_tiling(obj), + true); + + fenceable = (vma->node.size == fence_size && + (vma->node.start & (fence_alignment - 1)) == 0); + + mappable = (vma->node.start + fence_size <= + dev_priv->ggtt.mappable_end); + + /* + * Explicitly disable for rotated VMA since the display does not + * need the fence and the VMA is not accessible to other users. + */ + if (mappable && fenceable && + vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED) + vma->flags |= I915_VMA_CAN_FENCE; + else + vma->flags &= ~I915_VMA_CAN_FENCE; +} + +bool i915_gem_valid_gtt_space(struct i915_vma *vma, + unsigned long cache_level) +{ + struct drm_mm_node *gtt_space = &vma->node; + struct drm_mm_node *other; + + /* + * On some machines we have to be careful when putting differing types + * of snoopable memory together to avoid the prefetcher crossing memory + * domains and dying. During vm initialisation, we decide whether or not + * these constraints apply and set the drm_mm.color_adjust + * appropriately. + */ + if (vma->vm->mm.color_adjust == NULL) + return true; + + if (!drm_mm_node_allocated(gtt_space)) + return true; + + if (list_empty(>t_space->node_list)) + return true; + + other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); + if (other->allocated && !other->hole_follows && other->color != cache_level) + return false; + + other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); + if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) + return false; + + return true; +} + +/** + * i915_vma_insert - finds a slot for the vma in its address space + * @vma: the vma + * @size: requested size in bytes (can be larger than the VMA) + * @alignment: required alignment + * @flags: mask of PIN_* flags to use + * + * First we try to allocate some free space that meets the requirements for + * the VMA. Failiing that, if the flags permit, it will evict an old VMA, + * preferrably the oldest idle entry to make room for the new VMA. + * + * Returns: + * 0 on success, negative error code otherwise. + */ +static int +i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) +{ + struct drm_i915_private *dev_priv = to_i915(vma->vm->dev); + struct drm_i915_gem_object *obj = vma->obj; + u64 start, end; + int ret; + + GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); + GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); + + size = max(size, vma->size); + if (flags & PIN_MAPPABLE) + size = i915_gem_get_ggtt_size(dev_priv, size, + i915_gem_object_get_tiling(obj)); + + alignment = max(max(alignment, vma->display_alignment), + i915_gem_get_ggtt_alignment(dev_priv, size, + i915_gem_object_get_tiling(obj), + flags & PIN_MAPPABLE)); + + start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; + + end = vma->vm->total; + if (flags & PIN_MAPPABLE) + end = min_t(u64, end, dev_priv->ggtt.mappable_end); + if (flags & PIN_ZONE_4G) + end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE); + + /* If binding the object/GGTT view requires more space than the entire + * aperture has, reject it early before evicting everything in a vain + * attempt to find space. + */ + if (size > end) { + DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n", + size, obj->base.size, + flags & PIN_MAPPABLE ? "mappable" : "total", + end); + return -E2BIG; + } + + ret = i915_gem_object_pin_pages(obj); + if (ret) + return ret; + + if (flags & PIN_OFFSET_FIXED) { + u64 offset = flags & PIN_OFFSET_MASK; + if (offset & (alignment - 1) || offset > end - size) { + ret = -EINVAL; + goto err_unpin; + } + + vma->node.start = offset; + vma->node.size = size; + vma->node.color = obj->cache_level; + ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); + if (ret) { + ret = i915_gem_evict_for_vma(vma); + if (ret == 0) + ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); + if (ret) + goto err_unpin; + } + } else { + u32 search_flag, alloc_flag; + + if (flags & PIN_HIGH) { + search_flag = DRM_MM_SEARCH_BELOW; + alloc_flag = DRM_MM_CREATE_TOP; + } else { + search_flag = DRM_MM_SEARCH_DEFAULT; + alloc_flag = DRM_MM_CREATE_DEFAULT; + } + + /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks, + * so we know that we always have a minimum alignment of 4096. + * The drm_mm range manager is optimised to return results + * with zero alignment, so where possible use the optimal + * path. + */ + if (alignment <= 4096) + alignment = 0; + +search_free: + ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm, + &vma->node, + size, alignment, + obj->cache_level, + start, end, + search_flag, + alloc_flag); + if (ret) { + ret = i915_gem_evict_something(vma->vm, size, alignment, + obj->cache_level, + start, end, + flags); + if (ret == 0) + goto search_free; + + goto err_unpin; + } + + GEM_BUG_ON(vma->node.start < start); + GEM_BUG_ON(vma->node.start + vma->node.size > end); + } + GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); + + list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); + list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + obj->bind_count++; + GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); + + return 0; + +err_unpin: + i915_gem_object_unpin_pages(obj); + return ret; +} + +int __i915_vma_do_pin(struct i915_vma *vma, + u64 size, u64 alignment, u64 flags) +{ + unsigned int bound = vma->flags; + int ret; + + lockdep_assert_held(&vma->vm->dev->struct_mutex); + GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); + GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); + + if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) { + ret = -EBUSY; + goto err; + } + + if ((bound & I915_VMA_BIND_MASK) == 0) { + ret = i915_vma_insert(vma, size, alignment, flags); + if (ret) + goto err; + } + + ret = i915_vma_bind(vma, vma->obj->cache_level, flags); + if (ret) + goto err; + + if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND) + __i915_vma_set_map_and_fenceable(vma); + + GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); + return 0; + +err: + __i915_vma_unpin(vma); + return ret; +} + +void i915_vma_destroy(struct i915_vma *vma) +{ + GEM_BUG_ON(vma->node.allocated); + GEM_BUG_ON(i915_vma_is_active(vma)); + GEM_BUG_ON(!i915_vma_is_closed(vma)); + GEM_BUG_ON(vma->fence); + + list_del(&vma->vm_link); + if (!i915_vma_is_ggtt(vma)) + i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm)); + + kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma); +} + +void i915_vma_close(struct i915_vma *vma) +{ + GEM_BUG_ON(i915_vma_is_closed(vma)); + vma->flags |= I915_VMA_CLOSED; + + list_del(&vma->obj_link); + rb_erase(&vma->obj_node, &vma->obj->vma_tree); + + if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma)) + WARN_ON(i915_vma_unbind(vma)); +} + +static void __i915_vma_iounmap(struct i915_vma *vma) +{ + GEM_BUG_ON(i915_vma_is_pinned(vma)); + + if (vma->iomap == NULL) + return; + + io_mapping_unmap(vma->iomap); + vma->iomap = NULL; +} + +int i915_vma_unbind(struct i915_vma *vma) +{ + struct drm_i915_gem_object *obj = vma->obj; + unsigned long active; + int ret; + + lockdep_assert_held(&obj->base.dev->struct_mutex); + + /* First wait upon any activity as retiring the request may + * have side-effects such as unpinning or even unbinding this vma. + */ + active = i915_vma_get_active(vma); + if (active) { + int idx; + + /* When a closed VMA is retired, it is unbound - eek. + * In order to prevent it from being recursively closed, + * take a pin on the vma so that the second unbind is + * aborted. + * + * Even more scary is that the retire callback may free + * the object (last active vma). To prevent the explosion + * we defer the actual object free to a worker that can + * only proceed once it acquires the struct_mutex (which + * we currently hold, therefore it cannot free this object + * before we are finished). + */ + __i915_vma_pin(vma); + + for_each_active(active, idx) { + ret = i915_gem_active_retire(&vma->last_read[idx], + &vma->vm->dev->struct_mutex); + if (ret) + break; + } + + __i915_vma_unpin(vma); + if (ret) + return ret; + + GEM_BUG_ON(i915_vma_is_active(vma)); + } + + if (i915_vma_is_pinned(vma)) + return -EBUSY; + + if (!drm_mm_node_allocated(&vma->node)) + goto destroy; + + GEM_BUG_ON(obj->bind_count == 0); + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + + if (i915_vma_is_map_and_fenceable(vma)) { + /* release the fence reg _after_ flushing */ + ret = i915_vma_put_fence(vma); + if (ret) + return ret; + + /* Force a pagefault for domain tracking on next user access */ + i915_gem_release_mmap(obj); + + __i915_vma_iounmap(vma); + vma->flags &= ~I915_VMA_CAN_FENCE; + } + + if (likely(!vma->vm->closed)) { + trace_i915_vma_unbind(vma); + vma->vm->unbind_vma(vma); + } + vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); + + drm_mm_remove_node(&vma->node); + list_move_tail(&vma->vm_link, &vma->vm->unbound_list); + + if (vma->pages != obj->mm.pages) { + GEM_BUG_ON(!vma->pages); + sg_free_table(vma->pages); + kfree(vma->pages); + } + vma->pages = NULL; + + /* Since the unbound list is global, only move to that list if + * no more VMAs exist. */ + if (--obj->bind_count == 0) + list_move_tail(&obj->global_link, + &to_i915(obj->base.dev)->mm.unbound_list); + + /* And finally now the object is completely decoupled from this vma, + * we can drop its hold on the backing storage and allow it to be + * reaped by the shrinker. + */ + i915_gem_object_unpin_pages(obj); + +destroy: + if (unlikely(i915_vma_is_closed(vma))) + i915_vma_destroy(vma); + + return 0; +} + diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h new file mode 100644 index 000000000000..85446f0b0b3f --- /dev/null +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -0,0 +1,341 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#ifndef __I915_VMA_H__ +#define __I915_VMA_H__ + +#include <linux/io-mapping.h> + +#include <drm/drm_mm.h> + +#include "i915_gem_gtt.h" +#include "i915_gem_fence_reg.h" +#include "i915_gem_object.h" +#include "i915_gem_request.h" + + +enum i915_cache_level; + +/** + * A VMA represents a GEM BO that is bound into an address space. Therefore, a + * VMA's presence cannot be guaranteed before binding, or after unbinding the + * object into/from the address space. + * + * To make things as simple as possible (ie. no refcounting), a VMA's lifetime + * will always be <= an objects lifetime. So object refcounting should cover us. + */ +struct i915_vma { + struct drm_mm_node node; + struct drm_i915_gem_object *obj; + struct i915_address_space *vm; + struct drm_i915_fence_reg *fence; + struct sg_table *pages; + void __iomem *iomap; + u64 size; + u64 display_alignment; + + unsigned int flags; + /** + * How many users have pinned this object in GTT space. The following + * users can each hold at most one reference: pwrite/pread, execbuffer + * (objects are not allowed multiple times for the same batchbuffer), + * and the framebuffer code. When switching/pageflipping, the + * framebuffer code has at most two buffers pinned per crtc. + * + * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 + * bits with absolutely no headroom. So use 4 bits. + */ +#define I915_VMA_PIN_MASK 0xf +#define I915_VMA_PIN_OVERFLOW BIT(5) + + /** Flags and address space this VMA is bound to */ +#define I915_VMA_GLOBAL_BIND BIT(6) +#define I915_VMA_LOCAL_BIND BIT(7) +#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW) + +#define I915_VMA_GGTT BIT(8) +#define I915_VMA_CAN_FENCE BIT(9) +#define I915_VMA_CLOSED BIT(10) + + unsigned int active; + struct i915_gem_active last_read[I915_NUM_ENGINES]; + struct i915_gem_active last_fence; + + /** + * Support different GGTT views into the same object. + * This means there can be multiple VMA mappings per object and per VM. + * i915_ggtt_view_type is used to distinguish between those entries. + * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also + * assumed in GEM functions which take no ggtt view parameter. + */ + struct i915_ggtt_view ggtt_view; + + /** This object's place on the active/inactive lists */ + struct list_head vm_link; + + struct list_head obj_link; /* Link in the object's VMA list */ + struct rb_node obj_node; + + /** This vma's place in the batchbuffer or on the eviction list */ + struct list_head exec_list; + + /** + * Used for performing relocations during execbuffer insertion. + */ + struct hlist_node exec_node; + unsigned long exec_handle; + struct drm_i915_gem_exec_object2 *exec_entry; +}; + +struct i915_vma * +i915_vma_create(struct drm_i915_gem_object *obj, + struct i915_address_space *vm, + const struct i915_ggtt_view *view); + +void i915_vma_unpin_and_release(struct i915_vma **p_vma); + +static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) +{ + return vma->flags & I915_VMA_GGTT; +} + +static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma) +{ + return vma->flags & I915_VMA_CAN_FENCE; +} + +static inline bool i915_vma_is_closed(const struct i915_vma *vma) +{ + return vma->flags & I915_VMA_CLOSED; +} + +static inline unsigned int i915_vma_get_active(const struct i915_vma *vma) +{ + return vma->active; +} + +static inline bool i915_vma_is_active(const struct i915_vma *vma) +{ + return i915_vma_get_active(vma); +} + +static inline void i915_vma_set_active(struct i915_vma *vma, + unsigned int engine) +{ + vma->active |= BIT(engine); +} + +static inline void i915_vma_clear_active(struct i915_vma *vma, + unsigned int engine) +{ + vma->active &= ~BIT(engine); +} + +static inline bool i915_vma_has_active_engine(const struct i915_vma *vma, + unsigned int engine) +{ + return vma->active & BIT(engine); +} + +static inline u32 i915_ggtt_offset(const struct i915_vma *vma) +{ + GEM_BUG_ON(!i915_vma_is_ggtt(vma)); + GEM_BUG_ON(!vma->node.allocated); + GEM_BUG_ON(upper_32_bits(vma->node.start)); + GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1)); + return lower_32_bits(vma->node.start); +} + +static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) +{ + i915_gem_object_get(vma->obj); + return vma; +} + +static inline void i915_vma_put(struct i915_vma *vma) +{ + i915_gem_object_put(vma->obj); +} + +static inline long +i915_vma_compare(struct i915_vma *vma, + struct i915_address_space *vm, + const struct i915_ggtt_view *view) +{ + GEM_BUG_ON(view && !i915_is_ggtt(vm)); + + if (vma->vm != vm) + return vma->vm - vm; + + if (!view) + return vma->ggtt_view.type; + + if (vma->ggtt_view.type != view->type) + return vma->ggtt_view.type - view->type; + + return memcmp(&vma->ggtt_view.params, + &view->params, + sizeof(view->params)); +} + +int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, + u32 flags); +bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level); +bool +i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags); +void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); +int __must_check i915_vma_unbind(struct i915_vma *vma); +void i915_vma_close(struct i915_vma *vma); +void i915_vma_destroy(struct i915_vma *vma); + +int __i915_vma_do_pin(struct i915_vma *vma, + u64 size, u64 alignment, u64 flags); +static inline int __must_check +i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) +{ + BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW); + BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); + BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); + + /* Pin early to prevent the shrinker/eviction logic from destroying + * our vma as we insert and bind. + */ + if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) + return 0; + + return __i915_vma_do_pin(vma, size, alignment, flags); +} + +static inline int i915_vma_pin_count(const struct i915_vma *vma) +{ + return vma->flags & I915_VMA_PIN_MASK; +} + +static inline bool i915_vma_is_pinned(const struct i915_vma *vma) +{ + return i915_vma_pin_count(vma); +} + +static inline void __i915_vma_pin(struct i915_vma *vma) +{ + vma->flags++; + GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW); +} + +static inline void __i915_vma_unpin(struct i915_vma *vma) +{ + GEM_BUG_ON(!i915_vma_is_pinned(vma)); + vma->flags--; +} + +static inline void i915_vma_unpin(struct i915_vma *vma) +{ + GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); + __i915_vma_unpin(vma); +} + +/** + * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture + * @vma: VMA to iomap + * + * The passed in VMA has to be pinned in the global GTT mappable region. + * An extra pinning of the VMA is acquired for the return iomapping, + * the caller must call i915_vma_unpin_iomap to relinquish the pinning + * after the iomapping is no longer required. + * + * Callers must hold the struct_mutex. + * + * Returns a valid iomapped pointer or ERR_PTR. + */ +void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); +#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x)) + +/** + * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap + * @vma: VMA to unpin + * + * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). + * + * Callers must hold the struct_mutex. This function is only valid to be + * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap(). + */ +static inline void i915_vma_unpin_iomap(struct i915_vma *vma) +{ + lockdep_assert_held(&vma->vm->dev->struct_mutex); + GEM_BUG_ON(vma->iomap == NULL); + i915_vma_unpin(vma); +} + +static inline struct page *i915_vma_first_page(struct i915_vma *vma) +{ + GEM_BUG_ON(!vma->pages); + return sg_page(vma->pages->sgl); +} + +/** + * i915_vma_pin_fence - pin fencing state + * @vma: vma to pin fencing for + * + * This pins the fencing state (whether tiled or untiled) to make sure the + * vma (and its object) is ready to be used as a scanout target. Fencing + * status must be synchronize first by calling i915_vma_get_fence(): + * + * The resulting fence pin reference must be released again with + * i915_vma_unpin_fence(). + * + * Returns: + * + * True if the vma has a fence, false otherwise. + */ +static inline bool +i915_vma_pin_fence(struct i915_vma *vma) +{ + lockdep_assert_held(&vma->vm->dev->struct_mutex); + if (vma->fence) { + vma->fence->pin_count++; + return true; + } else + return false; +} + +/** + * i915_vma_unpin_fence - unpin fencing state + * @vma: vma to unpin fencing for + * + * This releases the fence pin reference acquired through + * i915_vma_pin_fence. It will handle both objects with and without an + * attached fence correctly, callers do not need to distinguish this. + */ +static inline void +i915_vma_unpin_fence(struct i915_vma *vma) +{ + lockdep_assert_held(&vma->vm->dev->struct_mutex); + if (vma->fence) { + GEM_BUG_ON(vma->fence->pin_count <= 0); + vma->fence->pin_count--; + } +} + +#endif + diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c index c762ae549a1c..dbe9fb41ae53 100644 --- a/drivers/gpu/drm/i915/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/intel_atomic_plane.c @@ -84,7 +84,6 @@ intel_plane_duplicate_state(struct drm_plane *plane) state = &intel_state->base; __drm_atomic_helper_plane_duplicate_state(plane, state); - intel_state->wait_req = NULL; return state; } @@ -101,13 +100,13 @@ void intel_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state) { - WARN_ON(state && to_intel_plane_state(state)->wait_req); drm_atomic_helper_plane_destroy_state(plane, state); } static int intel_plane_atomic_check(struct drm_plane *plane, struct drm_plane_state *state) { + struct drm_i915_private *dev_priv = to_i915(plane->dev); struct drm_crtc *crtc = state->crtc; struct intel_crtc *intel_crtc; struct intel_crtc_state *crtc_state; @@ -143,10 +142,10 @@ static int intel_plane_atomic_check(struct drm_plane *plane, crtc_state->base.enable ? crtc_state->pipe_src_h : 0; if (state->fb && drm_rotation_90_or_270(state->rotation)) { - char *format_name; + struct drm_format_name_buf format_name; - if (!(state->fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || - state->fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)) { + if (state->fb->modifier != I915_FORMAT_MOD_Y_TILED && + state->fb->modifier != I915_FORMAT_MOD_Yf_TILED) { DRM_DEBUG_KMS("Y/Yf tiling required for 90/270!\n"); return -EINVAL; } @@ -159,9 +158,9 @@ static int intel_plane_atomic_check(struct drm_plane *plane, switch (state->fb->pixel_format) { case DRM_FORMAT_C8: case DRM_FORMAT_RGB565: - format_name = drm_get_format_name(state->fb->pixel_format); - DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n", format_name); - kfree(format_name); + DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n", + drm_get_format_name(state->fb->pixel_format, + &format_name)); return -EINVAL; default: @@ -169,6 +168,14 @@ static int intel_plane_atomic_check(struct drm_plane *plane, } } + /* CHV ignores the mirror bit when the rotate bit is set :( */ + if (IS_CHERRYVIEW(dev_priv) && + state->rotation & DRM_ROTATE_180 && + state->rotation & DRM_REFLECT_X) { + DRM_DEBUG_KMS("Cannot rotate and reflect at the same time\n"); + return -EINVAL; + } + intel_state->base.visible = false; ret = intel_plane->check_plane(plane, crtc_state, intel_state); if (ret) diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c index 7093cfbb62b1..49f10538d4aa 100644 --- a/drivers/gpu/drm/i915/intel_audio.c +++ b/drivers/gpu/drm/i915/intel_audio.c @@ -57,6 +57,63 @@ * struct &i915_audio_component_audio_ops @audio_ops is called from i915 driver. */ +/* DP N/M table */ +#define LC_540M 540000 +#define LC_270M 270000 +#define LC_162M 162000 + +struct dp_aud_n_m { + int sample_rate; + int clock; + u16 m; + u16 n; +}; + +/* Values according to DP 1.4 Table 2-104 */ +static const struct dp_aud_n_m dp_aud_n_m[] = { + { 32000, LC_162M, 1024, 10125 }, + { 44100, LC_162M, 784, 5625 }, + { 48000, LC_162M, 512, 3375 }, + { 64000, LC_162M, 2048, 10125 }, + { 88200, LC_162M, 1568, 5625 }, + { 96000, LC_162M, 1024, 3375 }, + { 128000, LC_162M, 4096, 10125 }, + { 176400, LC_162M, 3136, 5625 }, + { 192000, LC_162M, 2048, 3375 }, + { 32000, LC_270M, 1024, 16875 }, + { 44100, LC_270M, 784, 9375 }, + { 48000, LC_270M, 512, 5625 }, + { 64000, LC_270M, 2048, 16875 }, + { 88200, LC_270M, 1568, 9375 }, + { 96000, LC_270M, 1024, 5625 }, + { 128000, LC_270M, 4096, 16875 }, + { 176400, LC_270M, 3136, 9375 }, + { 192000, LC_270M, 2048, 5625 }, + { 32000, LC_540M, 1024, 33750 }, + { 44100, LC_540M, 784, 18750 }, + { 48000, LC_540M, 512, 11250 }, + { 64000, LC_540M, 2048, 33750 }, + { 88200, LC_540M, 1568, 18750 }, + { 96000, LC_540M, 1024, 11250 }, + { 128000, LC_540M, 4096, 33750 }, + { 176400, LC_540M, 3136, 18750 }, + { 192000, LC_540M, 2048, 11250 }, +}; + +static const struct dp_aud_n_m * +audio_config_dp_get_n_m(struct intel_crtc *intel_crtc, int rate) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) { + if (rate == dp_aud_n_m[i].sample_rate && + intel_crtc->config->port_clock == dp_aud_n_m[i].clock) + return &dp_aud_n_m[i]; + } + + return NULL; +} + static const struct { int clock; u32 config; @@ -225,16 +282,43 @@ hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port, const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); + struct i915_audio_component *acomp = dev_priv->audio_component; + int rate = acomp ? acomp->aud_sample_rate[port] : 0; + const struct dp_aud_n_m *nm = audio_config_dp_get_n_m(intel_crtc, rate); enum pipe pipe = intel_crtc->pipe; u32 tmp; + if (nm) + DRM_DEBUG_KMS("using Maud %u, Naud %u\n", nm->m, nm->n); + else + DRM_DEBUG_KMS("using automatic Maud, Naud\n"); + tmp = I915_READ(HSW_AUD_CFG(pipe)); tmp &= ~AUD_CONFIG_N_VALUE_INDEX; tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; tmp &= ~AUD_CONFIG_N_PROG_ENABLE; tmp |= AUD_CONFIG_N_VALUE_INDEX; + if (nm) { + tmp &= ~AUD_CONFIG_N_MASK; + tmp |= AUD_CONFIG_N(nm->n); + tmp |= AUD_CONFIG_N_PROG_ENABLE; + } + I915_WRITE(HSW_AUD_CFG(pipe), tmp); + + tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe)); + tmp &= ~AUD_CONFIG_M_MASK; + tmp &= ~AUD_M_CTS_M_VALUE_INDEX; + tmp &= ~AUD_M_CTS_M_PROG_ENABLE; + + if (nm) { + tmp |= nm->m; + tmp |= AUD_M_CTS_M_VALUE_INDEX; + tmp |= AUD_M_CTS_M_PROG_ENABLE; + } + + I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp); } static void @@ -254,19 +338,27 @@ hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port, tmp &= ~AUD_CONFIG_N_PROG_ENABLE; tmp |= audio_config_hdmi_pixel_clock(adjusted_mode); - if (adjusted_mode->crtc_clock == TMDS_296M || - adjusted_mode->crtc_clock == TMDS_297M) { - n = audio_config_hdmi_get_n(adjusted_mode, rate); - if (n != 0) { - tmp &= ~AUD_CONFIG_N_MASK; - tmp |= AUD_CONFIG_N(n); - tmp |= AUD_CONFIG_N_PROG_ENABLE; - } else { - DRM_DEBUG_KMS("no suitable N value is found\n"); - } + n = audio_config_hdmi_get_n(adjusted_mode, rate); + if (n != 0) { + DRM_DEBUG_KMS("using N %d\n", n); + + tmp &= ~AUD_CONFIG_N_MASK; + tmp |= AUD_CONFIG_N(n); + tmp |= AUD_CONFIG_N_PROG_ENABLE; + } else { + DRM_DEBUG_KMS("using automatic N\n"); } I915_WRITE(HSW_AUD_CFG(pipe), tmp); + + /* + * Let's disable "Enable CTS or M Prog bit" + * and let HW calculate the value + */ + tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe)); + tmp &= ~AUD_M_CTS_M_PROG_ENABLE; + tmp &= ~AUD_M_CTS_M_VALUE_INDEX; + I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp); } static void @@ -485,23 +577,26 @@ static void ilk_audio_codec_enable(struct drm_connector *connector, /** * intel_audio_codec_enable - Enable the audio codec for HD audio * @intel_encoder: encoder on which to enable audio + * @crtc_state: pointer to the current crtc state. + * @conn_state: pointer to the current connector state. * * The enable sequences may only be performed after enabling the transcoder and * port, and after completed link training. */ -void intel_audio_codec_enable(struct intel_encoder *intel_encoder) +void intel_audio_codec_enable(struct intel_encoder *intel_encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { struct drm_encoder *encoder = &intel_encoder->base; - struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); - const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; struct drm_connector *connector; struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct i915_audio_component *acomp = dev_priv->audio_component; enum port port = intel_encoder->port; - enum pipe pipe = crtc->pipe; + enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe; - connector = drm_select_eld(encoder); - if (!connector) + connector = conn_state->connector; + if (!connector || !connector->eld[0]) return; DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", @@ -512,7 +607,7 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder) /* ELD Conn_Type */ connector->eld[5] &= ~(3 << 2); - if (intel_crtc_has_dp_encoder(crtc->config)) + if (intel_crtc_has_dp_encoder(crtc_state)) connector->eld[5] |= (1 << 2); connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2; @@ -687,7 +782,8 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port, /* 1. get the pipe */ intel_encoder = get_saved_enc(dev_priv, port, pipe); if (!intel_encoder || !intel_encoder->base.crtc || - intel_encoder->type != INTEL_OUTPUT_HDMI) { + (intel_encoder->type != INTEL_OUTPUT_HDMI && + intel_encoder->type != INTEL_OUTPUT_DP)) { DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port)); err = -ENODEV; goto unlock; diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c index 5ab646ef8c9f..7ffab1abc518 100644 --- a/drivers/gpu/drm/i915/intel_bios.c +++ b/drivers/gpu/drm/i915/intel_bios.c @@ -1147,7 +1147,7 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, if (!child) return; - aux_channel = child->raw[25]; + aux_channel = child->common.aux_channel; ddc_pin = child->common.ddc_pin; is_dvi = child->common.device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING; @@ -1677,7 +1677,8 @@ bool intel_bios_is_port_edp(struct drm_i915_private *dev_priv, enum port port) return false; } -bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum port port) +static bool child_dev_is_dp_dual_mode(const union child_device_config *p_child, + enum port port) { static const struct { u16 dp, hdmi; @@ -1691,22 +1692,35 @@ bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum por [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, }, [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, }, }; - int i; if (port == PORT_A || port >= ARRAY_SIZE(port_mapping)) return false; - if (!dev_priv->vbt.child_dev_num) + if ((p_child->common.device_type & DEVICE_TYPE_DP_DUAL_MODE_BITS) != + (DEVICE_TYPE_DP_DUAL_MODE & DEVICE_TYPE_DP_DUAL_MODE_BITS)) return false; + if (p_child->common.dvo_port == port_mapping[port].dp) + return true; + + /* Only accept a HDMI dvo_port as DP++ if it has an AUX channel */ + if (p_child->common.dvo_port == port_mapping[port].hdmi && + p_child->common.aux_channel != 0) + return true; + + return false; +} + +bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, + enum port port) +{ + int i; + for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { const union child_device_config *p_child = &dev_priv->vbt.child_dev[i]; - if ((p_child->common.dvo_port == port_mapping[port].dp || - p_child->common.dvo_port == port_mapping[port].hdmi) && - (p_child->common.device_type & DEVICE_TYPE_DP_DUAL_MODE_BITS) == - (DEVICE_TYPE_DP_DUAL_MODE & DEVICE_TYPE_DP_DUAL_MODE_BITS)) + if (child_dev_is_dp_dual_mode(p_child, port)) return true; } diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index 56efcc507ea2..c9c46a538edb 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -83,16 +83,18 @@ static void irq_enable(struct intel_engine_cs *engine) */ engine->breadcrumbs.irq_posted = true; - spin_lock_irq(&engine->i915->irq_lock); + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); engine->irq_enable(engine); - spin_unlock_irq(&engine->i915->irq_lock); + spin_unlock(&engine->i915->irq_lock); } static void irq_disable(struct intel_engine_cs *engine) { - spin_lock_irq(&engine->i915->irq_lock); + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); engine->irq_disable(engine); - spin_unlock_irq(&engine->i915->irq_lock); + spin_unlock(&engine->i915->irq_lock); engine->breadcrumbs.irq_posted = false; } @@ -293,9 +295,9 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine, struct intel_breadcrumbs *b = &engine->breadcrumbs; bool first; - spin_lock(&b->lock); + spin_lock_irq(&b->lock); first = __intel_engine_add_wait(engine, wait); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); return first; } @@ -326,7 +328,7 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine, if (RB_EMPTY_NODE(&wait->node)) return; - spin_lock(&b->lock); + spin_lock_irq(&b->lock); if (RB_EMPTY_NODE(&wait->node)) goto out_unlock; @@ -400,7 +402,7 @@ out_unlock: GEM_BUG_ON(rb_first(&b->waiters) != (b->first_wait ? &b->first_wait->node : NULL)); GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh) ^ RB_EMPTY_ROOT(&b->waiters)); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static bool signal_complete(struct drm_i915_gem_request *request) @@ -473,14 +475,14 @@ static int intel_breadcrumbs_signaler(void *arg) * we just completed - so double check we are still * the oldest before picking the next one. */ - spin_lock(&b->lock); + spin_lock_irq(&b->lock); if (request == b->first_signal) { struct rb_node *rb = rb_next(&request->signaling.node); b->first_signal = rb ? to_signaler(rb) : NULL; } rb_erase(&request->signaling.node, &b->signals); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); i915_gem_request_put(request); } else { @@ -502,11 +504,20 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) struct rb_node *parent, **p; bool first, wakeup; - /* locked by dma_fence_enable_sw_signaling() */ + /* Note that we may be called from an interrupt handler on another + * device (e.g. nouveau signaling a fence completion causing us + * to submit a request, and so enable signaling). As such, + * we need to make sure that all other users of b->lock protect + * against interrupts, i.e. use spin_lock_irqsave. + */ + + /* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */ assert_spin_locked(&request->lock); + if (!request->global_seqno) + return; request->signaling.wait.tsk = b->signaler; - request->signaling.wait.seqno = request->fence.seqno; + request->signaling.wait.seqno = request->global_seqno; i915_gem_request_get(request); spin_lock(&b->lock); @@ -530,8 +541,8 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) p = &b->signals.rb_node; while (*p) { parent = *p; - if (i915_seqno_passed(request->fence.seqno, - to_signaler(parent)->fence.seqno)) { + if (i915_seqno_passed(request->global_seqno, + to_signaler(parent)->global_seqno)) { p = &parent->rb_right; first = false; } else { @@ -592,7 +603,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) struct intel_breadcrumbs *b = &engine->breadcrumbs; cancel_fake_irq(engine); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); __intel_breadcrumbs_disable_irq(b); if (intel_engine_has_waiter(engine)) { @@ -605,7 +616,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) irq_disable(engine); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) @@ -618,35 +629,28 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) cancel_fake_irq(engine); } -unsigned int intel_kick_waiters(struct drm_i915_private *i915) +unsigned int intel_breadcrumbs_busy(struct drm_i915_private *i915) { struct intel_engine_cs *engine; enum intel_engine_id id; unsigned int mask = 0; - /* To avoid the task_struct disappearing beneath us as we wake up - * the process, we must first inspect the task_struct->state under the - * RCU lock, i.e. as we call wake_up_process() we must be holding the - * rcu_read_lock(). - */ - for_each_engine(engine, i915, id) - if (unlikely(intel_engine_wakeup(engine))) - mask |= intel_engine_flag(engine); + for_each_engine(engine, i915, id) { + struct intel_breadcrumbs *b = &engine->breadcrumbs; - return mask; -} + spin_lock_irq(&b->lock); -unsigned int intel_kick_signalers(struct drm_i915_private *i915) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - unsigned int mask = 0; + if (b->first_wait) { + wake_up_process(b->first_wait->tsk); + mask |= intel_engine_flag(engine); + } - for_each_engine(engine, i915, id) { - if (unlikely(READ_ONCE(engine->breadcrumbs.first_signal))) { - wake_up_process(engine->breadcrumbs.signaler); + if (b->first_signal) { + wake_up_process(b->signaler); mask |= intel_engine_flag(engine); } + + spin_unlock_irq(&b->lock); } return mask; diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c index 445108855275..d81232b79f00 100644 --- a/drivers/gpu/drm/i915/intel_color.c +++ b/drivers/gpu/drm/i915/intel_color.c @@ -95,8 +95,7 @@ static void ctm_mult_by_limited(uint64_t *result, int64_t *input) static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state) { struct drm_crtc *crtc = crtc_state->crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int i, pipe = intel_crtc->pipe; uint16_t coeffs[9] = { 0, }; @@ -180,7 +179,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state) I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0); I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0); - if (INTEL_INFO(dev)->gen > 6) { + if (INTEL_GEN(dev_priv) > 6) { uint16_t postoff = 0; if (intel_crtc_state->limited_color_range) @@ -345,11 +344,10 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state) static void broadwell_load_luts(struct drm_crtc_state *state) { struct drm_crtc *crtc = state->crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc_state *intel_state = to_intel_crtc_state(state); enum pipe pipe = to_intel_crtc(crtc)->pipe; - uint32_t i, lut_size = INTEL_INFO(dev)->color.degamma_lut_size; + uint32_t i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; if (crtc_state_is_legacy(state)) { haswell_load_luts(state); @@ -428,8 +426,7 @@ static void broadwell_load_luts(struct drm_crtc_state *state) static void cherryview_load_luts(struct drm_crtc_state *state) { struct drm_crtc *crtc = state->crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); enum pipe pipe = to_intel_crtc(crtc)->pipe; struct drm_color_lut *lut; uint32_t i, lut_size; @@ -446,7 +443,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state) if (state->degamma_lut) { lut = (struct drm_color_lut *) state->degamma_lut->data; - lut_size = INTEL_INFO(dev)->color.degamma_lut_size; + lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; for (i = 0; i < lut_size; i++) { /* Write LUT in U0.14 format. */ word0 = @@ -461,7 +458,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state) if (state->gamma_lut) { lut = (struct drm_color_lut *) state->gamma_lut->data; - lut_size = INTEL_INFO(dev)->color.gamma_lut_size; + lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; for (i = 0; i < lut_size; i++) { /* Write LUT in U0.10 format. */ word0 = @@ -497,12 +494,12 @@ void intel_color_load_luts(struct drm_crtc_state *crtc_state) int intel_color_check(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state) { - struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); size_t gamma_length, degamma_length; - degamma_length = INTEL_INFO(dev)->color.degamma_lut_size * + degamma_length = INTEL_INFO(dev_priv)->color.degamma_lut_size * sizeof(struct drm_color_lut); - gamma_length = INTEL_INFO(dev)->color.gamma_lut_size * + gamma_length = INTEL_INFO(dev_priv)->color.gamma_lut_size * sizeof(struct drm_color_lut); /* @@ -529,8 +526,7 @@ int intel_color_check(struct drm_crtc *crtc, void intel_color_init(struct drm_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); drm_mode_crtc_set_gamma_size(crtc, 256); @@ -549,10 +545,10 @@ void intel_color_init(struct drm_crtc *crtc) } /* Enable color management support when we have degamma & gamma LUTs. */ - if (INTEL_INFO(dev)->color.degamma_lut_size != 0 && - INTEL_INFO(dev)->color.gamma_lut_size != 0) + if (INTEL_INFO(dev_priv)->color.degamma_lut_size != 0 && + INTEL_INFO(dev_priv)->color.gamma_lut_size != 0) drm_crtc_enable_color_mgmt(crtc, - INTEL_INFO(dev)->color.degamma_lut_size, - true, - INTEL_INFO(dev)->color.gamma_lut_size); + INTEL_INFO(dev_priv)->color.degamma_lut_size, + true, + INTEL_INFO(dev_priv)->color.gamma_lut_size); } diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index a97151fcb9f4..86ecec5601d4 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -147,14 +147,13 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state, int mode) { - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crt *crt = intel_encoder_to_crt(encoder); struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; u32 adpa; - if (INTEL_INFO(dev)->gen >= 5) + if (INTEL_GEN(dev_priv) >= 5) adpa = ADPA_HOTPLUG_BITS; else adpa = 0; @@ -573,7 +572,7 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe) POSTING_READ(pipeconf_reg); /* Wait for next Vblank to substitue * border color for Color info */ - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); st00 = I915_READ8(_VGA_MSR_WRITE); status = ((st00 & (1 << 4)) != 0) ? connector_status_connected : @@ -673,8 +672,7 @@ static const struct dmi_system_id intel_spurious_crt_detect[] = { static enum drm_connector_status intel_crt_detect(struct drm_connector *connector, bool force) { - struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_crt *crt = intel_attached_crt(connector); struct intel_encoder *intel_encoder = &crt->base; enum intel_display_power_domain power_domain; @@ -693,7 +691,7 @@ intel_crt_detect(struct drm_connector *connector, bool force) power_domain = intel_display_port_power_domain(intel_encoder); intel_display_power_get(dev_priv, power_domain); - if (I915_HAS_HOTPLUG(dev)) { + if (I915_HAS_HOTPLUG(dev_priv)) { /* We can not rely on the HPD pin always being correctly wired * up, for example many KVM do not pass it through, and so * only trust an assertion that the monitor is connected. @@ -715,7 +713,7 @@ intel_crt_detect(struct drm_connector *connector, bool force) * broken monitor (without edid) to work behind a broken kvm (that fails * to have the right resistors for HP detection) needs to fix this up. * For now just bail out. */ - if (I915_HAS_HOTPLUG(dev) && !i915.load_detect_test) { + if (I915_HAS_HOTPLUG(dev_priv) && !i915.load_detect_test) { status = connector_status_disconnected; goto out; } @@ -731,7 +729,7 @@ intel_crt_detect(struct drm_connector *connector, bool force) if (intel_get_load_detect_pipe(connector, NULL, &tmp, &ctx)) { if (intel_crt_detect_ddc(connector)) status = connector_status_connected; - else if (INTEL_INFO(dev)->gen < 4) + else if (INTEL_GEN(dev_priv) < 4) status = intel_crt_load_detect(crt, to_intel_crtc(connector->state->crtc)->pipe); else if (i915.load_detect_test) @@ -793,11 +791,10 @@ static int intel_crt_set_property(struct drm_connector *connector, void intel_crt_reset(struct drm_encoder *encoder) { - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_crt *crt = intel_encoder_to_crt(to_intel_encoder(encoder)); - if (INTEL_INFO(dev)->gen >= 5) { + if (INTEL_GEN(dev_priv) >= 5) { u32 adpa; adpa = I915_READ(crt->adpa_reg); @@ -915,7 +912,7 @@ void intel_crt_init(struct drm_device *dev) crt->base.disable = intel_disable_crt; } crt->base.enable = intel_enable_crt; - if (I915_HAS_HOTPLUG(dev) && + if (I915_HAS_HOTPLUG(dev_priv) && !dmi_check_system(intel_spurious_crt_detect)) crt->base.hpd_pin = HPD_CRT; if (HAS_DDI(dev_priv)) { @@ -932,7 +929,7 @@ void intel_crt_init(struct drm_device *dev) drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs); - if (!I915_HAS_HOTPLUG(dev)) + if (!I915_HAS_HOTPLUG(dev_priv)) intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT; /* diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c index 1ea0e1f43397..d7a04bca8c28 100644 --- a/drivers/gpu/drm/i915/intel_csr.c +++ b/drivers/gpu/drm/i915/intel_csr.c @@ -168,12 +168,6 @@ struct stepping_info { char substepping; }; -static const struct stepping_info kbl_stepping_info[] = { - {'A', '0'}, {'B', '0'}, {'C', '0'}, - {'D', '0'}, {'E', '0'}, {'F', '0'}, - {'G', '0'}, {'H', '0'}, {'I', '0'}, -}; - static const struct stepping_info skl_stepping_info[] = { {'A', '0'}, {'B', '0'}, {'C', '0'}, {'D', '0'}, {'E', '0'}, {'F', '0'}, @@ -194,10 +188,7 @@ intel_get_stepping_info(struct drm_i915_private *dev_priv) const struct stepping_info *si; unsigned int size; - if (IS_KABYLAKE(dev_priv)) { - size = ARRAY_SIZE(kbl_stepping_info); - si = kbl_stepping_info; - } else if (IS_SKYLAKE(dev_priv)) { + if (IS_SKYLAKE(dev_priv)) { size = ARRAY_SIZE(skl_stepping_info); si = skl_stepping_info; } else if (IS_BROXTON(dev_priv)) { diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index fb18d699ce10..10ec9d4b7d45 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -1547,7 +1547,6 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv, { const struct bxt_ddi_buf_trans *ddi_translations; u32 n_entries, i; - uint32_t val; if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) { n_entries = ARRAY_SIZE(bxt_ddi_translations_edp); @@ -1576,38 +1575,11 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv, } } - /* - * While we write to the group register to program all lanes at once we - * can read only lane registers and we pick lanes 0/1 for that. - */ - val = I915_READ(BXT_PORT_PCS_DW10_LN01(port)); - val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT); - I915_WRITE(BXT_PORT_PCS_DW10_GRP(port), val); - - val = I915_READ(BXT_PORT_TX_DW2_LN0(port)); - val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE); - val |= ddi_translations[level].margin << MARGIN_000_SHIFT | - ddi_translations[level].scale << UNIQ_TRANS_SCALE_SHIFT; - I915_WRITE(BXT_PORT_TX_DW2_GRP(port), val); - - val = I915_READ(BXT_PORT_TX_DW3_LN0(port)); - val &= ~SCALE_DCOMP_METHOD; - if (ddi_translations[level].enable) - val |= SCALE_DCOMP_METHOD; - - if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) - DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set"); - - I915_WRITE(BXT_PORT_TX_DW3_GRP(port), val); - - val = I915_READ(BXT_PORT_TX_DW4_LN0(port)); - val &= ~DE_EMPHASIS; - val |= ddi_translations[level].deemphasis << DEEMPH_SHIFT; - I915_WRITE(BXT_PORT_TX_DW4_GRP(port), val); - - val = I915_READ(BXT_PORT_PCS_DW10_LN01(port)); - val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT; - I915_WRITE(BXT_PORT_PCS_DW10_GRP(port), val); + bxt_ddi_phy_set_signal_level(dev_priv, port, + ddi_translations[level].margin, + ddi_translations[level].scale, + ddi_translations[level].enable, + ddi_translations[level].deemphasis); } static uint32_t translate_signal_level(int signal_levels) @@ -1781,8 +1753,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder, struct drm_connector_state *old_conn_state) { struct drm_encoder *encoder = &intel_encoder->base; - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->dev); enum port port = intel_ddi_get_encoder_port(intel_encoder); int type = intel_encoder->type; uint32_t val; @@ -1815,7 +1786,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder, if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) | DPLL_CTRL2_DDI_CLK_OFF(port))); - else if (INTEL_INFO(dev)->gen < 9) + else if (INTEL_GEN(dev_priv) < 9) I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE); if (type == INTEL_OUTPUT_HDMI) { @@ -1865,8 +1836,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder, struct drm_encoder *encoder = &intel_encoder->base; struct drm_crtc *crtc = encoder->crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->dev); enum port port = intel_ddi_get_encoder_port(intel_encoder); int type = intel_encoder->type; @@ -1884,7 +1854,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder, } else if (type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - if (port == PORT_A && INTEL_INFO(dev)->gen < 9) + if (port == PORT_A && INTEL_GEN(dev_priv) < 9) intel_dp_stop_link_train(intel_dp); intel_edp_backlight_on(intel_dp); @@ -1894,7 +1864,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder, if (intel_crtc->config->has_audio) { intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); - intel_audio_codec_enable(intel_encoder); + intel_audio_codec_enable(intel_encoder, pipe_config, conn_state); } } @@ -1923,332 +1893,14 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder, } } -bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, - enum dpio_phy phy) -{ - enum port port; - - if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy))) - return false; - - if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & - (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) { - DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n", - phy); - - return false; - } - - if (phy == DPIO_PHY1 && - !(I915_READ(BXT_PORT_REF_DW3(DPIO_PHY1)) & GRC_DONE)) { - DRM_DEBUG_DRIVER("DDI PHY 1 powered, but GRC isn't done\n"); - - return false; - } - - if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) { - DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n", - phy); - - return false; - } - - for_each_port_masked(port, - phy == DPIO_PHY0 ? BIT(PORT_B) | BIT(PORT_C) : - BIT(PORT_A)) { - u32 tmp = I915_READ(BXT_PHY_CTL(port)); - - if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) { - DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane " - "for port %c powered down " - "(PHY_CTL %08x)\n", - phy, port_name(port), tmp); - - return false; - } - } - - return true; -} - -static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) -{ - u32 val = I915_READ(BXT_PORT_REF_DW6(phy)); - - return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT; -} - -static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv, - enum dpio_phy phy) -{ - if (intel_wait_for_register(dev_priv, - BXT_PORT_REF_DW3(phy), - GRC_DONE, GRC_DONE, - 10)) - DRM_ERROR("timeout waiting for PHY%d GRC\n", phy); -} - -void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) -{ - u32 val; - - if (bxt_ddi_phy_is_enabled(dev_priv, phy)) { - /* Still read out the GRC value for state verification */ - if (phy == DPIO_PHY0) - dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy); - - if (bxt_ddi_phy_verify_state(dev_priv, phy)) { - DRM_DEBUG_DRIVER("DDI PHY %d already enabled, " - "won't reprogram it\n", phy); - - return; - } - - DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, " - "force reprogramming it\n", phy); - } - - val = I915_READ(BXT_P_CR_GT_DISP_PWRON); - val |= GT_DISPLAY_POWER_ON(phy); - I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); - - /* - * The PHY registers start out inaccessible and respond to reads with - * all 1s. Eventually they become accessible as they power up, then - * the reserved bit will give the default 0. Poll on the reserved bit - * becoming 0 to find when the PHY is accessible. - * HW team confirmed that the time to reach phypowergood status is - * anywhere between 50 us and 100us. - */ - if (wait_for_us(((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & - (PHY_RESERVED | PHY_POWER_GOOD)) == PHY_POWER_GOOD), 100)) { - DRM_ERROR("timeout during PHY%d power on\n", phy); - } - - /* Program PLL Rcomp code offset */ - val = I915_READ(BXT_PORT_CL1CM_DW9(phy)); - val &= ~IREF0RC_OFFSET_MASK; - val |= 0xE4 << IREF0RC_OFFSET_SHIFT; - I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val); - - val = I915_READ(BXT_PORT_CL1CM_DW10(phy)); - val &= ~IREF1RC_OFFSET_MASK; - val |= 0xE4 << IREF1RC_OFFSET_SHIFT; - I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val); - - /* Program power gating */ - val = I915_READ(BXT_PORT_CL1CM_DW28(phy)); - val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | - SUS_CLK_CONFIG; - I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val); - - if (phy == DPIO_PHY0) { - val = I915_READ(BXT_PORT_CL2CM_DW6_BC); - val |= DW6_OLDO_DYN_PWR_DOWN_EN; - I915_WRITE(BXT_PORT_CL2CM_DW6_BC, val); - } - - val = I915_READ(BXT_PORT_CL1CM_DW30(phy)); - val &= ~OCL2_LDOFUSE_PWR_DIS; - /* - * On PHY1 disable power on the second channel, since no port is - * connected there. On PHY0 both channels have a port, so leave it - * enabled. - * TODO: port C is only connected on BXT-P, so on BXT0/1 we should - * power down the second channel on PHY0 as well. - * - * FIXME: Clarify programming of the following, the register is - * read-only with bit 6 fixed at 0 at least in stepping A. - */ - if (phy == DPIO_PHY1) - val |= OCL2_LDOFUSE_PWR_DIS; - I915_WRITE(BXT_PORT_CL1CM_DW30(phy), val); - - if (phy == DPIO_PHY0) { - uint32_t grc_code; - /* - * PHY0 isn't connected to an RCOMP resistor so copy over - * the corresponding calibrated value from PHY1, and disable - * the automatic calibration on PHY0. - */ - val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, DPIO_PHY1); - grc_code = val << GRC_CODE_FAST_SHIFT | - val << GRC_CODE_SLOW_SHIFT | - val; - I915_WRITE(BXT_PORT_REF_DW6(DPIO_PHY0), grc_code); - - val = I915_READ(BXT_PORT_REF_DW8(DPIO_PHY0)); - val |= GRC_DIS | GRC_RDY_OVRD; - I915_WRITE(BXT_PORT_REF_DW8(DPIO_PHY0), val); - } - - val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); - val |= COMMON_RESET_DIS; - I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); - - if (phy == DPIO_PHY1) - bxt_phy_wait_grc_done(dev_priv, DPIO_PHY1); -} - -void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) -{ - uint32_t val; - - val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); - val &= ~COMMON_RESET_DIS; - I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); - - val = I915_READ(BXT_P_CR_GT_DISP_PWRON); - val &= ~GT_DISPLAY_POWER_ON(phy); - I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); -} - -static bool __printf(6, 7) -__phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, - i915_reg_t reg, u32 mask, u32 expected, - const char *reg_fmt, ...) -{ - struct va_format vaf; - va_list args; - u32 val; - - val = I915_READ(reg); - if ((val & mask) == expected) - return true; - - va_start(args, reg_fmt); - vaf.fmt = reg_fmt; - vaf.va = &args; - - DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: " - "current %08x, expected %08x (mask %08x)\n", - phy, &vaf, reg.reg, val, (val & ~mask) | expected, - mask); - - va_end(args); - - return false; -} - -bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, - enum dpio_phy phy) -{ - uint32_t mask; - bool ok; - -#define _CHK(reg, mask, exp, fmt, ...) \ - __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt, \ - ## __VA_ARGS__) - - if (!bxt_ddi_phy_is_enabled(dev_priv, phy)) - return false; - - ok = true; - - /* PLL Rcomp code offset */ - ok &= _CHK(BXT_PORT_CL1CM_DW9(phy), - IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT, - "BXT_PORT_CL1CM_DW9(%d)", phy); - ok &= _CHK(BXT_PORT_CL1CM_DW10(phy), - IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT, - "BXT_PORT_CL1CM_DW10(%d)", phy); - - /* Power gating */ - mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG; - ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask, - "BXT_PORT_CL1CM_DW28(%d)", phy); - - if (phy == DPIO_PHY0) - ok &= _CHK(BXT_PORT_CL2CM_DW6_BC, - DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN, - "BXT_PORT_CL2CM_DW6_BC"); - - /* - * TODO: Verify BXT_PORT_CL1CM_DW30 bit OCL2_LDOFUSE_PWR_DIS, - * at least on stepping A this bit is read-only and fixed at 0. - */ - - if (phy == DPIO_PHY0) { - u32 grc_code = dev_priv->bxt_phy_grc; - - grc_code = grc_code << GRC_CODE_FAST_SHIFT | - grc_code << GRC_CODE_SLOW_SHIFT | - grc_code; - mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK | - GRC_CODE_NOM_MASK; - ok &= _CHK(BXT_PORT_REF_DW6(DPIO_PHY0), mask, grc_code, - "BXT_PORT_REF_DW6(%d)", DPIO_PHY0); - - mask = GRC_DIS | GRC_RDY_OVRD; - ok &= _CHK(BXT_PORT_REF_DW8(DPIO_PHY0), mask, mask, - "BXT_PORT_REF_DW8(%d)", DPIO_PHY0); - } - - return ok; -#undef _CHK -} - -static uint8_t -bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - switch (pipe_config->lane_count) { - case 1: - return 0; - case 2: - return BIT(2) | BIT(0); - case 4: - return BIT(3) | BIT(2) | BIT(0); - default: - MISSING_CASE(pipe_config->lane_count); - - return 0; - } -} - static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); - enum port port = dport->port; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - int lane; - - for (lane = 0; lane < 4; lane++) { - u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane)); - - /* - * Note that on CHV this flag is called UPAR, but has - * the same function. - */ - val &= ~LATENCY_OPTIM; - if (intel_crtc->config->lane_lat_optim_mask & BIT(lane)) - val |= LATENCY_OPTIM; - - I915_WRITE(BXT_PORT_TX_DW14_LN(port, lane), val); - } -} - -static uint8_t -bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) -{ - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); - enum port port = dport->port; - int lane; - uint8_t mask; - - mask = 0; - for (lane = 0; lane < 4; lane++) { - u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane)); - - if (val & LATENCY_OPTIM) - mask |= BIT(lane); - } + uint8_t mask = intel_crtc->config->lane_lat_optim_mask; - return mask; + bxt_ddi_phy_set_lane_optim_mask(encoder, mask); } void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) @@ -2417,7 +2069,7 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder, if (IS_BROXTON(dev_priv) && ret) pipe_config->lane_lat_optim_mask = bxt_ddi_phy_calc_lane_lat_optim_mask(encoder, - pipe_config); + pipe_config->lane_count); return ret; diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index d6a8f11813d5..185e3bbc9ec9 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -282,12 +282,13 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) info->num_sprites[PIPE_A] = 2; info->num_sprites[PIPE_B] = 2; info->num_sprites[PIPE_C] = 1; - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { for_each_pipe(dev_priv, pipe) info->num_sprites[pipe] = 2; - else + } else if (INTEL_GEN(dev_priv) >= 5) { for_each_pipe(dev_priv, pipe) info->num_sprites[pipe] = 1; + } if (i915.disable_display) { DRM_INFO("Display disabled (module parameter)\n"); diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 895b3dc50e3f..cf5cff7b03b8 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -37,7 +37,6 @@ #include "intel_frontbuffer.h" #include <drm/i915_drm.h> #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "intel_dsi.h" #include "i915_trace.h" #include <drm/drm_atomic.h> @@ -116,8 +115,9 @@ static void chv_prepare_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config); static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); -static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc, - struct intel_crtc_state *crtc_state); +static void skl_init_scalers(struct drm_i915_private *dev_priv, + struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state); static void skylake_pfit_enable(struct intel_crtc *crtc); static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force); static void ironlake_pfit_enable(struct intel_crtc *crtc); @@ -1008,10 +1008,8 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, target_clock, refclk, NULL, best_clock); } -bool intel_crtc_active(struct drm_crtc *crtc) +bool intel_crtc_active(struct intel_crtc *crtc) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - /* Be paranoid as we can arrive here with only partial * state retrieved from the hardware during setup. * @@ -1025,22 +1023,20 @@ bool intel_crtc_active(struct drm_crtc *crtc) * crtc->state->active once we have proper CRTC states wired up * for atomic. */ - return intel_crtc->active && crtc->primary->state->fb && - intel_crtc->config->base.adjusted_mode.crtc_clock; + return crtc->active && crtc->base.primary->state->fb && + crtc->config->base.adjusted_mode.crtc_clock; } enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); - return intel_crtc->config->cpu_transcoder; + return crtc->config->cpu_transcoder; } -static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) +static bool pipe_dsl_stopped(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t reg = PIPEDSL(pipe); u32 line1, line2; u32 line_mask; @@ -1075,12 +1071,11 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) */ static void intel_wait_for_pipe_off(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; enum pipe pipe = crtc->pipe; - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { i915_reg_t reg = PIPECONF(cpu_transcoder); /* Wait for the Pipe State to go off */ @@ -1090,7 +1085,7 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc) WARN(1, "pipe_off wait timed out\n"); } else { /* Wait for the display line to settle */ - if (wait_for(pipe_dsl_stopped(dev, pipe), 100)) + if (wait_for(pipe_dsl_stopped(dev_priv, pipe), 100)) WARN(1, "pipe_off wait timed out\n"); } } @@ -1296,11 +1291,10 @@ static void assert_plane(struct drm_i915_private *dev_priv, static void assert_planes_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_device *dev = &dev_priv->drm; int i; /* Primary planes are fixed to pipes on gen4+ */ - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { u32 val = I915_READ(DSPCNTR(pipe)); I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE, "plane %c assertion failure, should be disabled but not\n", @@ -1322,10 +1316,9 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv, static void assert_sprites_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_device *dev = &dev_priv->drm; int sprite; - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { for_each_sprite(dev_priv, pipe, sprite) { u32 val = I915_READ(PLANE_CTL(pipe, sprite)); I915_STATE_WARN(val & PLANE_CTL_ENABLE, @@ -1339,12 +1332,12 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv, "sprite %c assertion failure, should be off on pipe %c but is still active\n", sprite_name(pipe, sprite), pipe_name(pipe)); } - } else if (INTEL_INFO(dev)->gen >= 7) { + } else if (INTEL_GEN(dev_priv) >= 7) { u32 val = I915_READ(SPRCTL(pipe)); I915_STATE_WARN(val & SPRITE_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", plane_name(pipe), pipe_name(pipe)); - } else if (INTEL_INFO(dev)->gen >= 5) { + } else if (INTEL_GEN(dev_priv) >= 5) { u32 val = I915_READ(DVSCNTR(pipe)); I915_STATE_WARN(val & DVS_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", @@ -1598,12 +1591,12 @@ static void chv_enable_pll(struct intel_crtc *crtc, } } -static int intel_num_dvo_pipes(struct drm_device *dev) +static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv) { struct intel_crtc *crtc; int count = 0; - for_each_intel_crtc(dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { count += crtc->base.state->active && intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO); } @@ -1613,8 +1606,7 @@ static int intel_num_dvo_pipes(struct drm_device *dev) static void i9xx_enable_pll(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); i915_reg_t reg = DPLL(crtc->pipe); u32 dpll = crtc->config->dpll_hw_state.dpll; @@ -1625,7 +1617,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) assert_panel_unlocked(dev_priv, crtc->pipe); /* Enable DVO 2x clock on both PLLs if necessary */ - if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev) > 0) { + if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) { /* * It appears to be important that we don't enable this * for the current pipe before otherwise configuring the @@ -1650,7 +1642,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) POSTING_READ(reg); udelay(150); - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { I915_WRITE(DPLL_MD(crtc->pipe), crtc->config->dpll_hw_state.dpll_md); } else { @@ -1685,14 +1677,13 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) */ static void i9xx_disable_pll(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; /* Disable DVO 2x clock on both PLLs if necessary */ if (IS_I830(dev_priv) && intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) && - !intel_num_dvo_pipes(dev)) { + !intel_num_dvo_pipes(dev_priv)) { I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE); I915_WRITE(DPLL(PIPE_A), @@ -1788,8 +1779,8 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); i915_reg_t reg; uint32_t val, pipeconf_val; @@ -2198,7 +2189,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation) WARN_ON(!mutex_is_locked(&dev->struct_mutex)); - alignment = intel_surf_alignment(dev_priv, fb->modifier[0]); + alignment = intel_surf_alignment(dev_priv, fb->modifier); intel_fill_fb_ggtt_view(&view, fb, rotation); @@ -2359,13 +2350,13 @@ static u32 intel_adjust_tile_offset(int *x, int *y, WARN_ON(new_offset > old_offset); - if (fb->modifier[plane] != DRM_FORMAT_MOD_NONE) { + if (fb->modifier != DRM_FORMAT_MOD_NONE) { unsigned int tile_size, tile_width, tile_height; unsigned int pitch_tiles; tile_size = intel_tile_size(dev_priv); intel_tile_dims(dev_priv, &tile_width, &tile_height, - fb->modifier[plane], cpp); + fb->modifier, cpp); if (drm_rotation_90_or_270(rotation)) { pitch_tiles = pitch / tile_height; @@ -2408,7 +2399,7 @@ static u32 _intel_compute_tile_offset(const struct drm_i915_private *dev_priv, unsigned int rotation, u32 alignment) { - uint64_t fb_modifier = fb->modifier[plane]; + uint64_t fb_modifier = fb->modifier; unsigned int cpp = drm_format_plane_cpp(fb->pixel_format, plane); u32 offset, offset_aligned; @@ -2467,7 +2458,7 @@ u32 intel_compute_tile_offset(int *x, int *y, if (fb->pixel_format == DRM_FORMAT_NV12 && plane == 1) alignment = 4096; else - alignment = intel_surf_alignment(dev_priv, fb->modifier[plane]); + alignment = intel_surf_alignment(dev_priv, fb->modifier); return _intel_compute_tile_offset(dev_priv, x, y, fb, plane, pitch, rotation, alignment); @@ -2549,13 +2540,13 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv, DRM_ROTATE_0, tile_size); offset /= tile_size; - if (fb->modifier[i] != DRM_FORMAT_MOD_NONE) { + if (fb->modifier != DRM_FORMAT_MOD_NONE) { unsigned int tile_width, tile_height; unsigned int pitch_tiles; struct drm_rect r; intel_tile_dims(dev_priv, &tile_width, &tile_height, - fb->modifier[i], cpp); + fb->modifier, cpp); rot_info->plane[i].offset = offset; rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp); @@ -2714,7 +2705,7 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc, mode_cmd.width = fb->width; mode_cmd.height = fb->height; mode_cmd.pitches[0] = fb->pitches[0]; - mode_cmd.modifier[0] = fb->modifier[0]; + mode_cmd.modifier[0] = fb->modifier; mode_cmd.flags = DRM_MODE_FB_MODIFIERS; if (intel_framebuffer_init(dev, to_intel_framebuffer(fb), @@ -2824,14 +2815,8 @@ valid_fb: plane_state->crtc_w = fb->width; plane_state->crtc_h = fb->height; - intel_state->base.src.x1 = plane_state->src_x; - intel_state->base.src.y1 = plane_state->src_y; - intel_state->base.src.x2 = plane_state->src_x + plane_state->src_w; - intel_state->base.src.y2 = plane_state->src_y + plane_state->src_h; - intel_state->base.dst.x1 = plane_state->crtc_x; - intel_state->base.dst.y1 = plane_state->crtc_y; - intel_state->base.dst.x2 = plane_state->crtc_x + plane_state->crtc_w; - intel_state->base.dst.y2 = plane_state->crtc_y + plane_state->crtc_h; + intel_state->base.src = drm_plane_state_src(plane_state); + intel_state->base.dst = drm_plane_state_dest(plane_state); obj = intel_fb_obj(fb); if (i915_gem_object_is_tiled(obj)) @@ -2850,7 +2835,7 @@ static int skl_max_plane_width(const struct drm_framebuffer *fb, int plane, { int cpp = drm_format_plane_cpp(fb->pixel_format, plane); - switch (fb->modifier[plane]) { + switch (fb->modifier) { case DRM_FORMAT_MOD_NONE: case I915_FORMAT_MOD_X_TILED: switch (cpp) { @@ -2881,7 +2866,7 @@ static int skl_max_plane_width(const struct drm_framebuffer *fb, int plane, } break; default: - MISSING_CASE(fb->modifier[plane]); + MISSING_CASE(fb->modifier); } return 2048; @@ -2909,7 +2894,7 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state) intel_add_fb_offsets(&x, &y, plane_state, 0); offset = intel_compute_tile_offset(&x, &y, plane_state, 0); - alignment = intel_surf_alignment(dev_priv, fb->modifier[0]); + alignment = intel_surf_alignment(dev_priv, fb->modifier); /* * AUX surface offset is specified as the distance from the @@ -2926,7 +2911,7 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state) * * TODO: linear and Y-tiled seem fine, Yf untested, */ - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) { + if (fb->modifier == I915_FORMAT_MOD_X_TILED) { int cpp = drm_format_plane_cpp(fb->pixel_format, 0); while ((x + w) * cpp > fb->pitches[0]) { @@ -3013,11 +2998,9 @@ static void i9xx_update_primary_plane(struct drm_plane *primary, const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { - struct drm_device *dev = primary->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(primary->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_framebuffer *fb = plane_state->base.fb; - struct drm_i915_gem_object *obj = intel_fb_obj(fb); int plane = intel_crtc->plane; u32 linear_offset; u32 dspcntr; @@ -3030,7 +3013,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary, dspcntr |= DISPLAY_PLANE_ENABLE; - if (INTEL_INFO(dev)->gen < 4) { + if (INTEL_GEN(dev_priv) < 4) { if (intel_crtc->pipe == PIPE_B) dspcntr |= DISPPLANE_SEL_PIPE_B; @@ -3076,28 +3059,34 @@ static void i9xx_update_primary_plane(struct drm_plane *primary, } if (INTEL_GEN(dev_priv) >= 4 && - fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + fb->modifier == I915_FORMAT_MOD_X_TILED) dspcntr |= DISPPLANE_TILED; + if (rotation & DRM_ROTATE_180) + dspcntr |= DISPPLANE_ROTATE_180; + + if (rotation & DRM_REFLECT_X) + dspcntr |= DISPPLANE_MIRROR; + if (IS_G4X(dev_priv)) dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; intel_add_fb_offsets(&x, &y, plane_state, 0); - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) intel_crtc->dspaddr_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); - if (rotation == DRM_ROTATE_180) { - dspcntr |= DISPPLANE_ROTATE_180; - - x += (crtc_state->pipe_src_w - 1); - y += (crtc_state->pipe_src_h - 1); + if (rotation & DRM_ROTATE_180) { + x += crtc_state->pipe_src_w - 1; + y += crtc_state->pipe_src_h - 1; + } else if (rotation & DRM_REFLECT_X) { + x += crtc_state->pipe_src_w - 1; } linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); - if (INTEL_INFO(dev)->gen < 4) + if (INTEL_GEN(dev_priv) < 4) intel_crtc->dspaddr_offset = linear_offset; intel_crtc->adjusted_x = x; @@ -3106,14 +3095,17 @@ static void i9xx_update_primary_plane(struct drm_plane *primary, I915_WRITE(reg, dspcntr); I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]); - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { I915_WRITE(DSPSURF(plane), intel_fb_gtt_offset(fb, rotation) + intel_crtc->dspaddr_offset); I915_WRITE(DSPTILEOFF(plane), (y << 16) | x); I915_WRITE(DSPLINOFF(plane), linear_offset); - } else - I915_WRITE(DSPADDR(plane), i915_gem_object_ggtt_offset(obj, NULL) + linear_offset); + } else { + I915_WRITE(DSPADDR(plane), + intel_fb_gtt_offset(fb, rotation) + + intel_crtc->dspaddr_offset); + } POSTING_READ(reg); } @@ -3178,9 +3170,12 @@ static void ironlake_update_primary_plane(struct drm_plane *primary, BUG(); } - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (fb->modifier == I915_FORMAT_MOD_X_TILED) dspcntr |= DISPPLANE_TILED; + if (rotation & DRM_ROTATE_180) + dspcntr |= DISPPLANE_ROTATE_180; + if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; @@ -3189,13 +3184,11 @@ static void ironlake_update_primary_plane(struct drm_plane *primary, intel_crtc->dspaddr_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); - if (rotation == DRM_ROTATE_180) { - dspcntr |= DISPPLANE_ROTATE_180; - - if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) { - x += (crtc_state->pipe_src_w - 1); - y += (crtc_state->pipe_src_h - 1); - } + /* HSW+ does this automagically in hardware */ + if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv) && + rotation & DRM_ROTATE_180) { + x += crtc_state->pipe_src_w - 1; + y += crtc_state->pipe_src_h - 1; } linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); @@ -3287,9 +3280,9 @@ u32 skl_plane_stride(const struct drm_framebuffer *fb, int plane, if (drm_rotation_90_or_270(rotation)) { int cpp = drm_format_plane_cpp(fb->pixel_format, plane); - stride /= intel_tile_height(dev_priv, fb->modifier[0], cpp); + stride /= intel_tile_height(dev_priv, fb->modifier, cpp); } else { - stride /= intel_fb_stride_alignment(dev_priv, fb->modifier[0], + stride /= intel_fb_stride_alignment(dev_priv, fb->modifier, fb->pixel_format); } @@ -3385,9 +3378,6 @@ static void skylake_update_primary_plane(struct drm_plane *plane, struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_framebuffer *fb = plane_state->base.fb; - const struct skl_wm_values *wm = &dev_priv->wm.skl_results; - const struct skl_plane_wm *p_wm = - &crtc_state->wm.skl.optimal.planes[0]; int pipe = intel_crtc->pipe; u32 plane_ctl; unsigned int rotation = plane_state->base.rotation; @@ -3408,7 +3398,7 @@ static void skylake_update_primary_plane(struct drm_plane *plane, PLANE_CTL_PIPE_CSC_ENABLE; plane_ctl |= skl_plane_ctl_format(fb->pixel_format); - plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]); + plane_ctl |= skl_plane_ctl_tiling(fb->modifier); plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE; plane_ctl |= skl_plane_ctl_rotation(rotation); @@ -3423,9 +3413,6 @@ static void skylake_update_primary_plane(struct drm_plane *plane, intel_crtc->adjusted_x = src_x; intel_crtc->adjusted_y = src_y; - if (wm->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) - skl_write_plane_wm(intel_crtc, p_wm, &wm->ddb, 0); - I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl); I915_WRITE(PLANE_OFFSET(pipe, 0), (src_y << 16) | src_x); I915_WRITE(PLANE_STRIDE(pipe, 0), stride); @@ -3458,18 +3445,8 @@ static void skylake_disable_primary_plane(struct drm_plane *primary, struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - const struct skl_plane_wm *p_wm = &cstate->wm.skl.optimal.planes[0]; int pipe = intel_crtc->pipe; - /* - * We only populate skl_results on watermark updates, and if the - * plane's visiblity isn't actually changing neither is its watermarks. - */ - if (!crtc->primary->state->visible) - skl_write_plane_wm(intel_crtc, p_wm, - &dev_priv->wm.skl_results.ddb, 0); - I915_WRITE(PLANE_CTL(pipe, 0), 0); I915_WRITE(PLANE_SURF(pipe, 0), 0); POSTING_READ(PLANE_SURF(pipe, 0)); @@ -3519,7 +3496,7 @@ __intel_display_resume(struct drm_device *dev, int i, ret; intel_modeset_setup_hw_state(dev); - i915_redisable_vga(dev); + i915_redisable_vga(to_i915(dev)); if (!state) return 0; @@ -3696,8 +3673,7 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) static void intel_update_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *old_crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc->base.state); @@ -3722,7 +3698,7 @@ static void intel_update_pipe_config(struct intel_crtc *crtc, (pipe_config->pipe_src_h - 1)); /* on skylake this is done by detaching scalers */ - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { skl_detach_scalers(crtc); if (pipe_config->pch_pfit.enabled) @@ -4253,6 +4229,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc) bool intel_has_pending_fb_unpin(struct drm_device *dev) { + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc; /* Note that we don't need to be called with mode_config.lock here @@ -4267,7 +4244,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) continue; if (crtc->flip_work) - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); return true; } @@ -4742,13 +4719,8 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, */ int skl_update_scaler_crtc(struct intel_crtc_state *state) { - struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc); const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode; - DRM_DEBUG_KMS("Updating scaler for [CRTC:%d:%s] scaler_user index %u.%u\n", - intel_crtc->base.base.id, intel_crtc->base.name, - intel_crtc->pipe, SKL_CRTC_INDEX); - return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX, &state->scaler_state.scaler_id, DRM_ROTATE_0, state->pipe_src_w, state->pipe_src_h, @@ -4769,7 +4741,6 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); struct intel_plane *intel_plane = to_intel_plane(plane_state->base.plane); struct drm_framebuffer *fb = plane_state->base.fb; @@ -4777,10 +4748,6 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, bool force_detach = !fb || !plane_state->base.visible; - DRM_DEBUG_KMS("Updating scaler for [PLANE:%d:%s] scaler_user index %u.%u\n", - intel_plane->base.base.id, intel_plane->base.name, - intel_crtc->pipe, drm_plane_index(&intel_plane->base)); - ret = skl_update_scaler(crtc_state, force_detach, drm_plane_index(&intel_plane->base), &plane_state->scaler_id, @@ -4944,7 +4911,7 @@ void hsw_disable_ips(struct intel_crtc *crtc) } /* We need to wait for a vblank before we can disable the plane. */ - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc) @@ -5056,7 +5023,7 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) if (HAS_GMCH_DISPLAY(dev_priv)) { intel_set_memory_cxsr(dev_priv, false); dev_priv->wm.vlv.cxsr = false; - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); } } @@ -5075,7 +5042,7 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state) crtc->wm.cxsr_allowed = true; if (pipe_config->update_wm_post && pipe_config->base.active) - intel_update_watermarks(&crtc->base); + intel_update_watermarks(crtc); if (old_pri_state) { struct intel_plane_state *primary_state = @@ -5104,6 +5071,8 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) struct drm_plane_state *old_pri_state = drm_atomic_get_existing_plane_state(old_state, primary); bool modeset = needs_modeset(&pipe_config->base); + struct intel_atomic_state *old_intel_state = + to_intel_atomic_state(old_state); if (old_pri_state) { struct intel_plane_state *primary_state = @@ -5133,7 +5102,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) if (old_crtc_state->base.active) { intel_set_memory_cxsr(dev_priv, false); dev_priv->wm.vlv.cxsr = false; - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } } @@ -5146,7 +5115,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) */ if (pipe_config->disable_lp_wm) { ilk_disable_lp_wm(dev); - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } /* @@ -5171,9 +5140,10 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) * us to. */ if (dev_priv->display.initial_watermarks != NULL) - dev_priv->display.initial_watermarks(pipe_config); + dev_priv->display.initial_watermarks(old_intel_state, + pipe_config); else if (pipe_config->update_wm_pre) - intel_update_watermarks(&crtc->base); + intel_update_watermarks(crtc); } static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask) @@ -5327,6 +5297,8 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; + struct intel_atomic_state *old_intel_state = + to_intel_atomic_state(old_state); if (WARN_ON(intel_crtc->active)) return; @@ -5385,7 +5357,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, intel_color_load_luts(&pipe_config->base); if (dev_priv->display.initial_watermarks != NULL) - dev_priv->display.initial_watermarks(intel_crtc->config); + dev_priv->display.initial_watermarks(old_intel_state, intel_crtc->config); intel_enable_pipe(intel_crtc); if (intel_crtc->config->has_pch_encoder) @@ -5401,7 +5373,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, /* Must wait for vblank to avoid spurious PCH FIFO underruns */ if (intel_crtc->config->has_pch_encoder) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); } @@ -5416,11 +5388,12 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, struct drm_atomic_state *old_state) { struct drm_crtc *crtc = pipe_config->base.crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe, hsw_workaround_pipe; enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; + struct intel_atomic_state *old_intel_state = + to_intel_atomic_state(old_state); if (WARN_ON(intel_crtc->active)) return; @@ -5475,7 +5448,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, if (!transcoder_is_dsi(cpu_transcoder)) intel_ddi_enable_pipe_clock(intel_crtc); - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) skylake_pfit_enable(intel_crtc); else ironlake_pfit_enable(intel_crtc); @@ -5491,9 +5464,10 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, intel_ddi_enable_transcoder_func(crtc); if (dev_priv->display.initial_watermarks != NULL) - dev_priv->display.initial_watermarks(pipe_config); + dev_priv->display.initial_watermarks(old_intel_state, + pipe_config); else - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); /* XXX: Do the pipe assertions at the right place for BXT DSI. */ if (!transcoder_is_dsi(cpu_transcoder)) @@ -5502,7 +5476,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, if (intel_crtc->config->has_pch_encoder) lpt_pch_enable(crtc); - if (intel_crtc->config->dp_encoder_is_mst) + if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST)) intel_ddi_set_vc_payload_alloc(crtc, true); assert_vblank_disabled(crtc); @@ -5511,8 +5485,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, intel_encoders_enable(crtc, pipe_config, old_state); if (intel_crtc->config->has_pch_encoder) { - intel_wait_for_vblank(dev, pipe); - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, true); @@ -5522,8 +5496,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, * to change the workaround. */ hsw_workaround_pipe = pipe_config->hsw_workaround_pipe; if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) { - intel_wait_for_vblank(dev, hsw_workaround_pipe); - intel_wait_for_vblank(dev, hsw_workaround_pipe); + intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); + intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); } } @@ -5607,8 +5581,7 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state, struct drm_atomic_state *old_state) { struct drm_crtc *crtc = old_crtc_state->base.crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; @@ -5625,13 +5598,13 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state, if (!transcoder_is_dsi(cpu_transcoder)) intel_disable_pipe(intel_crtc); - if (intel_crtc->config->dp_encoder_is_mst) + if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST)) intel_ddi_set_vc_payload_alloc(crtc, false); if (!transcoder_is_dsi(cpu_transcoder)) intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder); - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) skylake_scaler_disable(intel_crtc); else ironlake_pfit_disable(intel_crtc, false); @@ -5844,10 +5817,8 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) static int skl_calc_cdclk(int max_pixclk, int vco); -static void intel_update_max_cdclk(struct drm_device *dev) +static void intel_update_max_cdclk(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK; int max_cdclk, vco; @@ -5905,11 +5876,9 @@ static void intel_update_max_cdclk(struct drm_device *dev) dev_priv->max_dotclk_freq); } -static void intel_update_cdclk(struct drm_device *dev) +static void intel_update_cdclk(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev); + dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev_priv); if (INTEL_GEN(dev_priv) >= 9) DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n", @@ -6070,14 +6039,14 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, int cdclk) return; } - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); } static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) { u32 cdctl, expected; - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); if (dev_priv->cdclk_pll.vco == 0 || dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref) @@ -6210,7 +6179,7 @@ void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco) dev_priv->skl_preferred_vco_freq = vco; if (changed) - intel_update_max_cdclk(&dev_priv->drm); + intel_update_max_cdclk(dev_priv); } static void @@ -6296,7 +6265,6 @@ static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv) static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco) { - struct drm_device *dev = &dev_priv->drm; u32 freq_select, pcu_ack; WARN_ON((cdclk == 24000) != (vco == 0)); @@ -6347,7 +6315,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco) sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack); mutex_unlock(&dev_priv->rps.hw_lock); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); } static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv); @@ -6394,7 +6362,7 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0) goto sanitize; - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); /* Is PLL enabled and locked ? */ if (dev_priv->cdclk_pll.vco == 0 || dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref) @@ -6428,7 +6396,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) struct drm_i915_private *dev_priv = to_i915(dev); u32 val, cmd; - WARN_ON(dev_priv->display.get_display_clock_speed(dev) + WARN_ON(dev_priv->display.get_display_clock_speed(dev_priv) != dev_priv->cdclk_freq); if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */ @@ -6485,7 +6453,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) mutex_unlock(&dev_priv->sb_lock); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); } static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) @@ -6493,7 +6461,7 @@ static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) struct drm_i915_private *dev_priv = to_i915(dev); u32 val, cmd; - WARN_ON(dev_priv->display.get_display_clock_speed(dev) + WARN_ON(dev_priv->display.get_display_clock_speed(dev_priv) != dev_priv->cdclk_freq); switch (cdclk) { @@ -6526,7 +6494,7 @@ static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) } mutex_unlock(&dev_priv->rps.hw_lock); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); } static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv, @@ -6746,7 +6714,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config, intel_color_load_luts(&pipe_config->base); - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); intel_enable_pipe(intel_crtc); assert_vblank_disabled(crtc); @@ -6799,7 +6767,7 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config, intel_color_load_luts(&pipe_config->base); - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); intel_enable_pipe(intel_crtc); assert_vblank_disabled(crtc); @@ -6837,7 +6805,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state, * wait for planes to fully turn off before disabling the pipe. */ if (IS_GEN2(dev_priv)) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_encoders_disable(crtc, old_crtc_state, old_state); @@ -6915,7 +6883,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc) encoder->base.crtc = NULL; intel_fbc_disable(intel_crtc); - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); intel_disable_shared_dpll(intel_crtc); domains = intel_crtc->enabled_power_domains; @@ -7064,7 +7032,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, } } - if (INTEL_INFO(dev)->num_pipes == 2) + if (INTEL_INFO(dev_priv)->num_pipes == 2) return 0; /* Ivybridge 3 pipe is really complicated */ @@ -7075,7 +7043,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, if (pipe_config->fdi_lanes <= 2) return 0; - other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C)); + other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -7094,7 +7062,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, return -EINVAL; } - other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B)); + other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -7205,7 +7173,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; int clock_limit = dev_priv->max_dotclk_freq; - if (INTEL_INFO(dev)->gen < 4) { + if (INTEL_GEN(dev_priv) < 4) { clock_limit = dev_priv->max_cdclk_freq * 9 / 10; /* @@ -7252,10 +7220,9 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, return 0; } -static int skylake_get_display_clock_speed(struct drm_device *dev) +static int skylake_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t cdctl; + u32 cdctl; skl_dpll0_update(dev_priv); @@ -7314,9 +7281,8 @@ static void bxt_de_pll_update(struct drm_i915_private *dev_priv) dev_priv->cdclk_pll.ref; } -static int broxton_get_display_clock_speed(struct drm_device *dev) +static int broxton_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 divider; int div, vco; @@ -7349,9 +7315,8 @@ static int broxton_get_display_clock_speed(struct drm_device *dev) return DIV_ROUND_CLOSEST(vco, div); } -static int broadwell_get_display_clock_speed(struct drm_device *dev) +static int broadwell_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t lcpll = I915_READ(LCPLL_CTL); uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; @@ -7369,9 +7334,8 @@ static int broadwell_get_display_clock_speed(struct drm_device *dev) return 675000; } -static int haswell_get_display_clock_speed(struct drm_device *dev) +static int haswell_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t lcpll = I915_READ(LCPLL_CTL); uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; @@ -7387,35 +7351,35 @@ static int haswell_get_display_clock_speed(struct drm_device *dev) return 540000; } -static int valleyview_get_display_clock_speed(struct drm_device *dev) +static int valleyview_get_display_clock_speed(struct drm_i915_private *dev_priv) { - return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk", + return vlv_get_cck_clock_hpll(dev_priv, "cdclk", CCK_DISPLAY_CLOCK_CONTROL); } -static int ilk_get_display_clock_speed(struct drm_device *dev) +static int ilk_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 450000; } -static int i945_get_display_clock_speed(struct drm_device *dev) +static int i945_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 400000; } -static int i915_get_display_clock_speed(struct drm_device *dev) +static int i915_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 333333; } -static int i9xx_misc_get_display_clock_speed(struct drm_device *dev) +static int i9xx_misc_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 200000; } -static int pnv_get_display_clock_speed(struct drm_device *dev) +static int pnv_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; u16 gcfgc = 0; pci_read_config_word(pdev, GCFGC, &gcfgc); @@ -7438,9 +7402,9 @@ static int pnv_get_display_clock_speed(struct drm_device *dev) } } -static int i915gm_get_display_clock_speed(struct drm_device *dev) +static int i915gm_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; u16 gcfgc = 0; pci_read_config_word(pdev, GCFGC, &gcfgc); @@ -7458,14 +7422,14 @@ static int i915gm_get_display_clock_speed(struct drm_device *dev) } } -static int i865_get_display_clock_speed(struct drm_device *dev) +static int i865_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 266667; } -static int i85x_get_display_clock_speed(struct drm_device *dev) +static int i85x_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; u16 hpllcc = 0; /* @@ -7501,14 +7465,13 @@ static int i85x_get_display_clock_speed(struct drm_device *dev) return 0; } -static int i830_get_display_clock_speed(struct drm_device *dev) +static int i830_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 133333; } -static unsigned int intel_hpll_vco(struct drm_device *dev) +static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); static const unsigned int blb_vco[8] = { [0] = 3200000, [1] = 4000000, @@ -7555,16 +7518,16 @@ static unsigned int intel_hpll_vco(struct drm_device *dev) vco_table = ctg_vco; else if (IS_G4X(dev_priv)) vco_table = elk_vco; - else if (IS_CRESTLINE(dev)) + else if (IS_CRESTLINE(dev_priv)) vco_table = cl_vco; - else if (IS_PINEVIEW(dev)) + else if (IS_PINEVIEW(dev_priv)) vco_table = pnv_vco; - else if (IS_G33(dev)) + else if (IS_G33(dev_priv)) vco_table = blb_vco; else return 0; - tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO); + tmp = I915_READ(IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO); vco = vco_table[tmp & 0x7]; if (vco == 0) @@ -7575,10 +7538,10 @@ static unsigned int intel_hpll_vco(struct drm_device *dev) return vco; } -static int gm45_get_display_clock_speed(struct drm_device *dev) +static int gm45_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; - unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + struct pci_dev *pdev = dev_priv->drm.pdev; + unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv); uint16_t tmp = 0; pci_read_config_word(pdev, GCFGC, &tmp); @@ -7598,14 +7561,14 @@ static int gm45_get_display_clock_speed(struct drm_device *dev) } } -static int i965gm_get_display_clock_speed(struct drm_device *dev) +static int i965gm_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; static const uint8_t div_3200[] = { 16, 10, 8 }; static const uint8_t div_4000[] = { 20, 12, 10 }; static const uint8_t div_5333[] = { 24, 16, 14 }; const uint8_t *div_table; - unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv); uint16_t tmp = 0; pci_read_config_word(pdev, GCFGC, &tmp); @@ -7636,15 +7599,15 @@ fail: return 200000; } -static int g33_get_display_clock_speed(struct drm_device *dev) +static int g33_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; static const uint8_t div_3200[] = { 12, 10, 8, 7, 5, 16 }; static const uint8_t div_4000[] = { 14, 12, 10, 8, 6, 20 }; static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 }; static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 }; const uint8_t *div_table; - unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv); uint16_t tmp = 0; pci_read_config_word(pdev, GCFGC, &tmp); @@ -7733,10 +7696,10 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, struct dpll *reduced_clock) { - struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 fp, fp2 = 0; - if (IS_PINEVIEW(dev)) { + if (IS_PINEVIEW(dev_priv)) { fp = pnv_dpll_compute_fp(&crtc_state->dpll); if (reduced_clock) fp2 = pnv_dpll_compute_fp(reduced_clock); @@ -7804,12 +7767,11 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, struct intel_link_m_n *m_n, struct intel_link_m_n *m2_n2) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); int pipe = crtc->pipe; enum transcoder transcoder = crtc->config->cpu_transcoder; - if (INTEL_INFO(dev)->gen >= 5) { + if (INTEL_GEN(dev_priv) >= 5) { I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m); I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n); I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m); @@ -8106,11 +8068,10 @@ static void chv_prepare_pll(struct intel_crtc *crtc, * in cases where we need the PLL enabled even when @pipe is not going to * be enabled. */ -int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, +int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, const struct dpll *dpll) { - struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_crtc_state *pipe_config; pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL); @@ -8121,7 +8082,7 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, pipe_config->pixel_multiplier = 1; pipe_config->dpll = *dpll; - if (IS_CHERRYVIEW(to_i915(dev))) { + if (IS_CHERRYVIEW(dev_priv)) { chv_compute_dpll(crtc, pipe_config); chv_prepare_pll(crtc, pipe_config); chv_enable_pll(crtc, pipe_config); @@ -8144,20 +8105,19 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, * Disable the PLL for @pipe. To be used in cases where we need * the PLL enabled even when @pipe is not going to be enabled. */ -void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe) +void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe) { - if (IS_CHERRYVIEW(to_i915(dev))) - chv_disable_pll(to_i915(dev), pipe); + if (IS_CHERRYVIEW(dev_priv)) + chv_disable_pll(dev_priv, pipe); else - vlv_disable_pll(to_i915(dev), pipe); + vlv_disable_pll(dev_priv, pipe); } static void i9xx_compute_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, struct dpll *reduced_clock) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 dpll; struct dpll *clock = &crtc_state->dpll; @@ -8183,7 +8143,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLL_SDVO_HIGH_SPEED; /* compute bitmask from p1 value */ - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW; else { dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; @@ -8204,7 +8164,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; break; } - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT); if (crtc_state->sdvo_tv_clock) @@ -8218,7 +8178,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLL_VCO_ENABLE; crtc_state->dpll_hw_state.dpll = dpll; - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { u32 dpll_md = (crtc_state->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; crtc_state->dpll_hw_state.dpll_md = dpll_md; @@ -8265,8 +8225,7 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc, static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) { - struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); enum pipe pipe = intel_crtc->pipe; enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; @@ -8292,7 +8251,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) vsyncshift += adjusted_mode->crtc_htotal; } - if (INTEL_INFO(dev)->gen > 3) + if (INTEL_GEN(dev_priv) > 3) I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift); I915_WRITE(HTOTAL(cpu_transcoder), @@ -8415,8 +8374,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode, static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) { - struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); uint32_t pipeconf; pipeconf = 0; @@ -8452,7 +8410,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) } } - if (HAS_PIPE_CXSR(dev)) { + if (HAS_PIPE_CXSR(dev_priv)) { if (intel_crtc->lowfreq_avail) { DRM_DEBUG_KMS("enabling CxSR downclocking\n"); pipeconf |= PIPECONF_CXSR_DOWNCLOCK; @@ -8462,7 +8420,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) } if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { - if (INTEL_INFO(dev)->gen < 4 || + if (INTEL_GEN(dev_priv) < 4 || intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; else @@ -8670,8 +8628,7 @@ static int vlv_crtc_compute_clock(struct intel_crtc *crtc, static void i9xx_get_pfit_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); uint32_t tmp; if (INTEL_GEN(dev_priv) <= 3 && @@ -8683,7 +8640,7 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc, return; /* Check whether the pfit is attached to our pipe. */ - if (INTEL_INFO(dev)->gen < 4) { + if (INTEL_GEN(dev_priv) < 4) { if (crtc->pipe != PIPE_B) return; } else { @@ -8747,10 +8704,10 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc, fb = &intel_fb->base; - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { if (val & DISPPLANE_TILED) { plane_config->tiling = I915_TILING_X; - fb->modifier[0] = I915_FORMAT_MOD_X_TILED; + fb->modifier = I915_FORMAT_MOD_X_TILED; } } @@ -8759,7 +8716,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc, fb->pixel_format = fourcc; fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8; - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { if (plane_config->tiling) offset = I915_READ(DSPTILEOFF(plane)); else @@ -8779,7 +8736,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc, aligned_height = intel_fb_align_height(dev, fb->height, fb->pixel_format, - fb->modifier[0]); + fb->modifier); plane_config->size = fb->pitches[0] * aligned_height; @@ -8828,8 +8785,7 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc, static bool i9xx_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum intel_display_power_domain power_domain; uint32_t tmp; bool ret; @@ -8868,7 +8824,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, (tmp & PIPECONF_COLOR_RANGE_SELECT)) pipe_config->limited_color_range = true; - if (INTEL_INFO(dev)->gen < 4) + if (INTEL_GEN(dev_priv) < 4) pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE; intel_get_pipe_timings(crtc, pipe_config); @@ -8876,7 +8832,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, i9xx_get_pfit_config(crtc, pipe_config); - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { /* No way to read it out on pipes B and C */ if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A) tmp = dev_priv->chv_dpll_md[crtc->pipe]; @@ -9673,11 +9629,10 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, struct intel_link_m_n *m_n, struct intel_link_m_n *m2_n2) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - if (INTEL_INFO(dev)->gen >= 5) { + if (INTEL_GEN(dev_priv) >= 5) { m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder)); m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder)); m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder)) @@ -9689,7 +9644,7 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, * gen < 8) and if DRRS is supported (to make sure the * registers are not unnecessarily read). */ - if (m2_n2 && INTEL_INFO(dev)->gen < 8 && + if (m2_n2 && INTEL_GEN(dev_priv) < 8 && crtc->config->has_drrs) { m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder)); m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder)); @@ -9793,17 +9748,17 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc, tiling = val & PLANE_CTL_TILED_MASK; switch (tiling) { case PLANE_CTL_TILED_LINEAR: - fb->modifier[0] = DRM_FORMAT_MOD_NONE; + fb->modifier = DRM_FORMAT_MOD_NONE; break; case PLANE_CTL_TILED_X: plane_config->tiling = I915_TILING_X; - fb->modifier[0] = I915_FORMAT_MOD_X_TILED; + fb->modifier = I915_FORMAT_MOD_X_TILED; break; case PLANE_CTL_TILED_Y: - fb->modifier[0] = I915_FORMAT_MOD_Y_TILED; + fb->modifier = I915_FORMAT_MOD_Y_TILED; break; case PLANE_CTL_TILED_YF: - fb->modifier[0] = I915_FORMAT_MOD_Yf_TILED; + fb->modifier = I915_FORMAT_MOD_Yf_TILED; break; default: MISSING_CASE(tiling); @@ -9820,13 +9775,13 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc, fb->width = ((val >> 0) & 0x1fff) + 1; val = I915_READ(PLANE_STRIDE(pipe, 0)); - stride_mult = intel_fb_stride_alignment(dev_priv, fb->modifier[0], + stride_mult = intel_fb_stride_alignment(dev_priv, fb->modifier, fb->pixel_format); fb->pitches[0] = (val & 0x3ff) * stride_mult; aligned_height = intel_fb_align_height(dev, fb->height, fb->pixel_format, - fb->modifier[0]); + fb->modifier); plane_config->size = fb->pitches[0] * aligned_height; @@ -9891,10 +9846,10 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc, fb = &intel_fb->base; - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { if (val & DISPPLANE_TILED) { plane_config->tiling = I915_TILING_X; - fb->modifier[0] = I915_FORMAT_MOD_X_TILED; + fb->modifier = I915_FORMAT_MOD_X_TILED; } } @@ -9923,7 +9878,7 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc, aligned_height = intel_fb_align_height(dev, fb->height, fb->pixel_format, - fb->modifier[0]); + fb->modifier); plane_config->size = fb->pitches[0] * aligned_height; @@ -10191,7 +10146,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) } intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); } /* @@ -10261,6 +10216,29 @@ static void bxt_modeset_commit_cdclk(struct drm_atomic_state *old_state) bxt_set_cdclk(to_i915(dev), req_cdclk); } +static int bdw_adjust_min_pipe_pixel_rate(struct intel_crtc_state *crtc_state, + int pixel_rate) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + + /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ + if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled) + pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95); + + /* BSpec says "Do not use DisplayPort with CDCLK less than + * 432 MHz, audio enabled, port width x4, and link rate + * HBR2 (5.4 GHz), or else there may be audio corruption or + * screen corruption." + */ + if (intel_crtc_has_dp_encoder(crtc_state) && + crtc_state->has_audio && + crtc_state->port_clock >= 540000 && + crtc_state->lane_count == 4) + pixel_rate = max(432000, pixel_rate); + + return pixel_rate; +} + /* compute the max rate for new configuration */ static int ilk_max_pixel_rate(struct drm_atomic_state *state) { @@ -10286,9 +10264,9 @@ static int ilk_max_pixel_rate(struct drm_atomic_state *state) pixel_rate = ilk_pipe_pixel_rate(crtc_state); - /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ - if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled) - pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95); + if (IS_BROADWELL(dev_priv) || IS_GEN9(dev_priv)) + pixel_rate = bdw_adjust_min_pipe_pixel_rate(crtc_state, + pixel_rate); intel_state->min_pixclk[i] = pixel_rate; } @@ -10371,7 +10349,7 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk) I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); WARN(cdclk != dev_priv->cdclk_freq, "cdclk requested %d kHz but got %d kHz\n", @@ -10658,8 +10636,7 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc, static void haswell_get_ddi_port_state(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_shared_dpll *pll; enum port port; uint32_t tmp; @@ -10686,7 +10663,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc, * DDI E. So just check whether this pipe is wired to DDI E and whether * the PCH transcoder is on. */ - if (INTEL_INFO(dev)->gen < 9 && + if (INTEL_GEN(dev_priv) < 9 && (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) { pipe_config->has_pch_encoder = true; @@ -10701,8 +10678,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc, static bool haswell_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum intel_display_power_domain power_domain; unsigned long power_domain_mask; bool active; @@ -10735,11 +10711,9 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK; - if (INTEL_INFO(dev)->gen >= 9) { - skl_init_scalers(dev, crtc, pipe_config); - } + if (INTEL_GEN(dev_priv) >= 9) { + skl_init_scalers(dev_priv, crtc, pipe_config); - if (INTEL_INFO(dev)->gen >= 9) { pipe_config->scaler_state.scaler_id = -1; pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX); } @@ -10747,7 +10721,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe); if (intel_display_power_get_if_enabled(dev_priv, power_domain)) { power_domain_mask |= BIT(power_domain); - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) skylake_get_pfit_config(crtc, pipe_config); else ironlake_get_pfit_config(crtc, pipe_config); @@ -10841,16 +10815,9 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base, struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - const struct skl_wm_values *wm = &dev_priv->wm.skl_results; - const struct skl_plane_wm *p_wm = - &cstate->wm.skl.optimal.planes[PLANE_CURSOR]; int pipe = intel_crtc->pipe; uint32_t cntl = 0; - if (INTEL_GEN(dev_priv) >= 9 && wm->dirty_pipes & drm_crtc_mask(crtc)) - skl_write_cursor_wm(intel_crtc, p_wm, &wm->ddb); - if (plane_state && plane_state->base.visible) { cntl = MCURSOR_GAMMA_ENABLE; switch (plane_state->base.crtc_w) { @@ -10872,7 +10839,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base, if (HAS_DDI(dev_priv)) cntl |= CURSOR_PIPE_CSC_ENABLE; - if (plane_state->base.rotation == DRM_ROTATE_180) + if (plane_state->base.rotation & DRM_ROTATE_180) cntl |= CURSOR_ROTATE_180; } @@ -10918,7 +10885,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, /* ILK+ do this automagically */ if (HAS_GMCH_DISPLAY(dev_priv) && - plane_state->base.rotation == DRM_ROTATE_180) { + plane_state->base.rotation & DRM_ROTATE_180) { base += (plane_state->base.crtc_h * plane_state->base.crtc_w - 1) * 4; } @@ -11051,7 +11018,7 @@ intel_framebuffer_create_for_mode(struct drm_device *dev, fb = intel_framebuffer_create(dev, &mode_cmd, obj); if (IS_ERR(fb)) - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return fb; } @@ -11136,6 +11103,7 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, struct drm_encoder *encoder = &intel_encoder->base; struct drm_crtc *crtc = NULL; struct drm_device *dev = encoder->dev; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_framebuffer *fb; struct drm_mode_config *config = &dev->mode_config; struct drm_atomic_state *state = NULL, *restore_state = NULL; @@ -11288,7 +11256,7 @@ found: old->restore_state = restore_state; /* let the connector get through one full cycle before testing */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); return true; fail: @@ -11367,7 +11335,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, fp = pipe_config->dpll_hw_state.fp1; clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT; - if (IS_PINEVIEW(dev)) { + if (IS_PINEVIEW(dev_priv)) { clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1; clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT; } else { @@ -11376,7 +11344,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, } if (!IS_GEN2(dev_priv)) { - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >> DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW); else @@ -11398,7 +11366,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, return; } - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) port_clock = pnv_calc_dpll_params(refclk, &clock); else port_clock = i9xx_calc_dpll_params(refclk, &clock); @@ -11667,8 +11635,7 @@ static bool pageflip_finished(struct intel_crtc *crtc, void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; unsigned long flags; @@ -11681,12 +11648,12 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) * lost pageflips) so needs the full irqsave spinlocks. */ spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && !is_mmio_work(work) && - pageflip_finished(intel_crtc, work)) - page_flip_completed(intel_crtc); + pageflip_finished(crtc, work)) + page_flip_completed(crtc); spin_unlock_irqrestore(&dev->event_lock, flags); } @@ -11694,8 +11661,7 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; unsigned long flags; @@ -11708,12 +11674,12 @@ void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) * lost pageflips) so needs the full irqsave spinlocks. */ spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && is_mmio_work(work) && - pageflip_finished(intel_crtc, work)) - page_flip_completed(intel_crtc); + pageflip_finished(crtc, work)) + page_flip_completed(crtc); spin_unlock_irqrestore(&dev->event_lock, flags); } @@ -11818,7 +11784,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev, MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset | - intel_fb_modifier_to_tiling(fb->modifier[0])); + intel_fb_modifier_to_tiling(fb->modifier)); /* XXX Enabling the panel-fitter across page-flip is so far * untested on non-native modes, so ignore it for now. @@ -11851,7 +11817,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev, intel_ring_emit(ring, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0] | - intel_fb_modifier_to_tiling(fb->modifier[0])); + intel_fb_modifier_to_tiling(fb->modifier)); intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset); /* Contrary to the suggestions in the documentation, @@ -11957,7 +11923,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev, intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit); intel_ring_emit(ring, fb->pitches[0] | - intel_fb_modifier_to_tiling(fb->modifier[0])); + intel_fb_modifier_to_tiling(fb->modifier)); intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset); intel_ring_emit(ring, (MI_NOOP)); @@ -11967,8 +11933,6 @@ static int intel_gen7_queue_flip(struct drm_device *dev, static bool use_mmio_flip(struct intel_engine_cs *engine, struct drm_i915_gem_object *obj) { - struct reservation_object *resv; - /* * This is not being used for older platforms, because * non-availability of flip done interrupt forces us to use @@ -11990,12 +11954,7 @@ static bool use_mmio_flip(struct intel_engine_cs *engine, else if (i915.enable_execlists) return true; - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv && !reservation_object_test_signaled_rcu(resv, false)) - return true; - - return engine != i915_gem_active_get_engine(&obj->last_write, - &obj->base.dev->struct_mutex); + return engine != i915_gem_object_last_write_engine(obj); } static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, @@ -12010,7 +11969,7 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, ctl = I915_READ(PLANE_CTL(pipe, 0)); ctl &= ~PLANE_CTL_TILED_MASK; - switch (fb->modifier[0]) { + switch (fb->modifier) { case DRM_FORMAT_MOD_NONE: break; case I915_FORMAT_MOD_X_TILED: @@ -12023,7 +11982,7 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, ctl |= PLANE_CTL_TILED_YF; break; default: - MISSING_CASE(fb->modifier[0]); + MISSING_CASE(fb->modifier); } /* @@ -12048,7 +12007,7 @@ static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc, dspcntr = I915_READ(reg); - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (fb->modifier == I915_FORMAT_MOD_X_TILED) dspcntr |= DISPPLANE_TILED; else dspcntr &= ~DISPPLANE_TILED; @@ -12068,17 +12027,8 @@ static void intel_mmio_flip_work_func(struct work_struct *w) struct intel_framebuffer *intel_fb = to_intel_framebuffer(crtc->base.primary->fb); struct drm_i915_gem_object *obj = intel_fb->obj; - struct reservation_object *resv; - if (work->flip_queued_req) - WARN_ON(i915_wait_request(work->flip_queued_req, - 0, NULL, NO_WAITBOOST)); - - /* For framebuffer backed by dmabuf, wait for fence */ - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) - WARN_ON(reservation_object_wait_timeout_rcu(resv, false, false, - MAX_SCHEDULE_TIMEOUT) < 0); + WARN_ON(i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT, NULL) < 0); intel_pipe_update_start(crtc); @@ -12141,8 +12091,7 @@ static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv, void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; WARN_ON(!in_interrupt()); @@ -12151,19 +12100,19 @@ void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) return; spin_lock(&dev->event_lock); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && !is_mmio_work(work) && - __pageflip_stall_check_cs(dev_priv, intel_crtc, work)) { + __pageflip_stall_check_cs(dev_priv, crtc, work)) { WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n", - work->flip_queued_vblank, intel_crtc_get_vblank_counter(intel_crtc)); - page_flip_completed(intel_crtc); + work->flip_queued_vblank, intel_crtc_get_vblank_counter(crtc)); + page_flip_completed(crtc); work = NULL; } if (work != NULL && !is_mmio_work(work) && - intel_crtc_get_vblank_counter(intel_crtc) - work->flip_queued_vblank > 1) + intel_crtc_get_vblank_counter(crtc) - work->flip_queued_vblank > 1) intel_queue_rps_boost_for_request(work->flip_queued_req); spin_unlock(&dev->event_lock); } @@ -12203,7 +12152,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, * TILEOFF/LINOFF registers can't be changed via MI display flips. * Note that pitch changes could also affect these register. */ - if (INTEL_INFO(dev)->gen > 3 && + if (INTEL_GEN(dev_priv) > 3 && (fb->offsets[0] != crtc->primary->fb->offsets[0] || fb->pitches[0] != crtc->primary->fb->pitches[0])) return -EINVAL; @@ -12263,7 +12212,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, intel_crtc->reset_count = i915_reset_count(&dev_priv->gpu_error); if (i915_reset_in_progress_or_wedged(&dev_priv->gpu_error)) { ret = -EIO; - goto cleanup; + goto unlock; } atomic_inc(&intel_crtc->unpin_work_count); @@ -12273,14 +12222,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { engine = dev_priv->engine[BCS]; - if (fb->modifier[0] != old_fb->modifier[0]) + if (fb->modifier != old_fb->modifier) /* vlv: DISPLAY_FLIP fails to change tiling */ engine = NULL; } else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) { engine = dev_priv->engine[BCS]; - } else if (INTEL_INFO(dev)->gen >= 7) { - engine = i915_gem_active_get_engine(&obj->last_write, - &obj->base.dev->struct_mutex); + } else if (INTEL_GEN(dev_priv) >= 7) { + engine = i915_gem_object_last_write_engine(obj); if (engine == NULL || engine->id != RCS) engine = dev_priv->engine[BCS]; } else { @@ -12312,9 +12260,6 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, if (mmio_flip) { INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func); - - work->flip_queued_req = i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); queue_work(system_unbound_wq, &work->mmio_work); } else { request = i915_gem_request_alloc(engine, engine->last_context); @@ -12338,6 +12283,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, i915_add_request_no_flush(request); } + i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY); i915_gem_track_fb(intel_fb_obj(old_fb), obj, to_intel_plane(primary)->frontbuffer_bit); mutex_unlock(&dev->struct_mutex); @@ -12355,12 +12301,13 @@ cleanup_unpin: intel_unpin_fb_obj(fb, crtc->primary->state->rotation); cleanup_pending: atomic_dec(&intel_crtc->unpin_work_count); +unlock: mutex_unlock(&dev->struct_mutex); cleanup: crtc->primary->fb = old_fb; update_state_fb(crtc->primary); - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); drm_framebuffer_unreference(work->old_fb); spin_lock_irq(&dev->event_lock); @@ -12433,7 +12380,7 @@ static bool intel_wm_need_update(struct drm_plane *plane, if (!cur->base.fb || !new->base.fb) return false; - if (cur->base.fb->modifier[0] != new->base.fb->modifier[0] || + if (cur->base.fb->modifier != new->base.fb->modifier || cur->base.rotation != new->base.rotation || drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) || drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) || @@ -12539,7 +12486,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state, /* Pre-gen9 platforms need two-step watermark updates */ if ((pipe_config->update_wm_pre || pipe_config->update_wm_post) && - INTEL_INFO(dev)->gen < 9 && dev_priv->display.optimize_watermarks) + INTEL_GEN(dev_priv) < 9 && dev_priv->display.optimize_watermarks) to_intel_crtc_state(crtc_state)->wm.need_postvbl_update = true; if (visible || was_visible) @@ -12644,7 +12591,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, * old state and the new state. We can program these * immediately. */ - ret = dev_priv->display.compute_intermediate_wm(crtc->dev, + ret = dev_priv->display.compute_intermediate_wm(dev, intel_crtc, pipe_config); if (ret) { @@ -12656,7 +12603,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal; } - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { if (mode_changed) ret = skl_update_scaler_crtc(pipe_config); @@ -12769,6 +12716,16 @@ static void intel_dump_crtc_timings(const struct drm_display_mode *mode) mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags); } +static inline void +intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id, + unsigned int lane_count, struct intel_link_m_n *m_n) +{ + DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", + id, lane_count, + m_n->gmch_m, m_n->gmch_n, + m_n->link_m, m_n->link_n, m_n->tu); +} + static void intel_dump_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config, const char *context) @@ -12780,61 +12737,58 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, struct intel_plane_state *state; struct drm_framebuffer *fb; - DRM_DEBUG_KMS("[CRTC:%d:%s]%s config %p for pipe %c\n", - crtc->base.base.id, crtc->base.name, - context, pipe_config, pipe_name(crtc->pipe)); + DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n", + crtc->base.base.id, crtc->base.name, context); - DRM_DEBUG_KMS("cpu_transcoder: %s\n", transcoder_name(pipe_config->cpu_transcoder)); - DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n", + DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n", + transcoder_name(pipe_config->cpu_transcoder), pipe_config->pipe_bpp, pipe_config->dither); - DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", - pipe_config->has_pch_encoder, - pipe_config->fdi_lanes, - pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n, - pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n, - pipe_config->fdi_m_n.tu); - DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", - intel_crtc_has_dp_encoder(pipe_config), - pipe_config->lane_count, - pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n, - pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n, - pipe_config->dp_m_n.tu); - - DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n", - intel_crtc_has_dp_encoder(pipe_config), - pipe_config->lane_count, - pipe_config->dp_m2_n2.gmch_m, - pipe_config->dp_m2_n2.gmch_n, - pipe_config->dp_m2_n2.link_m, - pipe_config->dp_m2_n2.link_n, - pipe_config->dp_m2_n2.tu); + + if (pipe_config->has_pch_encoder) + intel_dump_m_n_config(pipe_config, "fdi", + pipe_config->fdi_lanes, + &pipe_config->fdi_m_n); + + if (intel_crtc_has_dp_encoder(pipe_config)) { + intel_dump_m_n_config(pipe_config, "dp m_n", + pipe_config->lane_count, &pipe_config->dp_m_n); + if (pipe_config->has_drrs) + intel_dump_m_n_config(pipe_config, "dp m2_n2", + pipe_config->lane_count, + &pipe_config->dp_m2_n2); + } DRM_DEBUG_KMS("audio: %i, infoframes: %i\n", - pipe_config->has_audio, - pipe_config->has_infoframe); + pipe_config->has_audio, pipe_config->has_infoframe); DRM_DEBUG_KMS("requested mode:\n"); drm_mode_debug_printmodeline(&pipe_config->base.mode); DRM_DEBUG_KMS("adjusted mode:\n"); drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode); intel_dump_crtc_timings(&pipe_config->base.adjusted_mode); - DRM_DEBUG_KMS("port clock: %d\n", pipe_config->port_clock); - DRM_DEBUG_KMS("pipe src size: %dx%d\n", + DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d\n", + pipe_config->port_clock, pipe_config->pipe_src_w, pipe_config->pipe_src_h); - DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n", - crtc->num_scalers, - pipe_config->scaler_state.scaler_users, - pipe_config->scaler_state.scaler_id); - DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n", - pipe_config->gmch_pfit.control, - pipe_config->gmch_pfit.pgm_ratios, - pipe_config->gmch_pfit.lvds_border_bits); - DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n", - pipe_config->pch_pfit.pos, - pipe_config->pch_pfit.size, - pipe_config->pch_pfit.enabled ? "enabled" : "disabled"); - DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled); - DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide); + + if (INTEL_GEN(dev_priv) >= 9) + DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n", + crtc->num_scalers, + pipe_config->scaler_state.scaler_users, + pipe_config->scaler_state.scaler_id); + + if (HAS_GMCH_DISPLAY(dev_priv)) + DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n", + pipe_config->gmch_pfit.control, + pipe_config->gmch_pfit.pgm_ratios, + pipe_config->gmch_pfit.lvds_border_bits); + else + DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n", + pipe_config->pch_pfit.pos, + pipe_config->pch_pfit.size, + enableddisabled(pipe_config->pch_pfit.enabled)); + + DRM_DEBUG_KMS("ips: %i, double wide: %i\n", + pipe_config->ips_enabled, pipe_config->double_wide); if (IS_BROXTON(dev_priv)) { DRM_DEBUG_KMS("dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x," @@ -12872,7 +12826,7 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, DRM_DEBUG_KMS("planes on this crtc\n"); list_for_each_entry(plane, &dev->mode_config.plane_list, head) { - char *format_name; + struct drm_format_name_buf format_name; intel_plane = to_intel_plane(plane); if (intel_plane->pipe != crtc->pipe) continue; @@ -12885,23 +12839,20 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, continue; } - format_name = drm_get_format_name(fb->pixel_format); - - DRM_DEBUG_KMS("[PLANE:%d:%s] enabled", - plane->base.id, plane->name); - DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = %s", - fb->base.id, fb->width, fb->height, format_name); - DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n", - state->scaler_id, - state->base.src.x1 >> 16, - state->base.src.y1 >> 16, - drm_rect_width(&state->base.src) >> 16, - drm_rect_height(&state->base.src) >> 16, - state->base.dst.x1, state->base.dst.y1, - drm_rect_width(&state->base.dst), - drm_rect_height(&state->base.dst)); - - kfree(format_name); + DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n", + plane->base.id, plane->name, + fb->base.id, fb->width, fb->height, + drm_get_format_name(fb->pixel_format, &format_name)); + if (INTEL_GEN(dev_priv) >= 9) + DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n", + state->scaler_id, + state->base.src.x1 >> 16, + state->base.src.y1 >> 16, + drm_rect_width(&state->base.src) >> 16, + drm_rect_height(&state->base.src) >> 16, + state->base.dst.x1, state->base.dst.y1, + drm_rect_width(&state->base.dst), + drm_rect_height(&state->base.dst)); } } @@ -13216,12 +13167,11 @@ intel_compare_link_m_n(const struct intel_link_m_n *m_n, } static bool -intel_pipe_config_compare(struct drm_device *dev, +intel_pipe_config_compare(struct drm_i915_private *dev_priv, struct intel_crtc_state *current_config, struct intel_crtc_state *pipe_config, bool adjust) { - struct drm_i915_private *dev_priv = to_i915(dev); bool ret = true; #define INTEL_ERR_OR_DBG_KMS(fmt, ...) \ @@ -13341,7 +13291,7 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_I(lane_count); PIPE_CONF_CHECK_X(lane_lat_optim_mask); - if (INTEL_INFO(dev)->gen < 8) { + if (INTEL_GEN(dev_priv) < 8) { PIPE_CONF_CHECK_M_N(dp_m_n); if (current_config->has_drrs) @@ -13390,7 +13340,7 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_X(gmch_pfit.control); /* pfit ratios are autocomputed by the hw on gen4+ */ - if (INTEL_INFO(dev)->gen < 4) + if (INTEL_GEN(dev_priv) < 4) PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios); PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits); @@ -13465,8 +13415,7 @@ static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv, static void verify_wm_state(struct drm_crtc *crtc, struct drm_crtc_state *new_state) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct skl_ddb_allocation hw_ddb, *sw_ddb; struct skl_pipe_wm hw_wm, *sw_wm; struct skl_plane_wm *hw_plane_wm, *sw_plane_wm; @@ -13475,17 +13424,17 @@ static void verify_wm_state(struct drm_crtc *crtc, const enum pipe pipe = intel_crtc->pipe; int plane, level, max_level = ilk_wm_max_level(dev_priv); - if (INTEL_INFO(dev)->gen < 9 || !new_state->active) + if (INTEL_GEN(dev_priv) < 9 || !new_state->active) return; skl_pipe_wm_get_hw_state(crtc, &hw_wm); - sw_wm = &intel_crtc->wm.active.skl; + sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal; skl_ddb_get_hw_state(dev_priv, &hw_ddb); sw_ddb = &dev_priv->wm.skl_hw.ddb; /* planes */ - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { hw_plane_wm = &hw_wm.planes[plane]; sw_plane_wm = &sw_wm->planes[plane]; @@ -13581,11 +13530,15 @@ static void verify_wm_state(struct drm_crtc *crtc, } static void -verify_connector_state(struct drm_device *dev, struct drm_crtc *crtc) +verify_connector_state(struct drm_device *dev, + struct drm_atomic_state *state, + struct drm_crtc *crtc) { struct drm_connector *connector; + struct drm_connector_state *old_conn_state; + int i; - drm_for_each_connector(connector, dev) { + for_each_connector_in_state(state, connector, old_conn_state, i) { struct drm_encoder *encoder = connector->encoder; struct drm_connector_state *state = connector->state; @@ -13700,7 +13653,7 @@ verify_crtc_state(struct drm_crtc *crtc, intel_pipe_config_sanity_check(dev_priv, pipe_config); sw_config = to_intel_crtc_state(crtc->state); - if (!intel_pipe_config_compare(dev, sw_config, + if (!intel_pipe_config_compare(dev_priv, sw_config, pipe_config, false)) { I915_STATE_WARN(1, "pipe state doesn't match!\n"); intel_dump_pipe_config(intel_crtc, pipe_config, @@ -13793,15 +13746,16 @@ verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc, static void intel_modeset_verify_crtc(struct drm_crtc *crtc, - struct drm_crtc_state *old_state, - struct drm_crtc_state *new_state) + struct drm_atomic_state *state, + struct drm_crtc_state *old_state, + struct drm_crtc_state *new_state) { if (!needs_modeset(new_state) && !to_intel_crtc_state(new_state)->update_pipe) return; verify_wm_state(crtc, new_state); - verify_connector_state(crtc->dev, crtc); + verify_connector_state(crtc->dev, state, crtc); verify_crtc_state(crtc, old_state, new_state); verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state); } @@ -13817,10 +13771,11 @@ verify_disabled_dpll_state(struct drm_device *dev) } static void -intel_modeset_verify_disabled(struct drm_device *dev) +intel_modeset_verify_disabled(struct drm_device *dev, + struct drm_atomic_state *state) { verify_encoder_state(dev); - verify_connector_state(dev, NULL); + verify_connector_state(dev, state, NULL); verify_disabled_dpll_state(dev); } @@ -14040,8 +13995,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state) DRM_DEBUG_KMS("New cdclk calculated to be atomic %u, actual %u\n", intel_state->cdclk, intel_state->dev_cdclk); - } else + } else { to_intel_atomic_state(state)->cdclk = dev_priv->atomic_cdclk_freq; + } intel_modeset_clear_plls(state); @@ -14118,7 +14074,7 @@ static int intel_atomic_check(struct drm_device *dev, } if (i915.fastboot && - intel_pipe_config_compare(dev, + intel_pipe_config_compare(dev_priv, to_intel_crtc_state(crtc->state), pipe_config, true)) { crtc_state->mode_changed = false; @@ -14142,8 +14098,9 @@ static int intel_atomic_check(struct drm_device *dev, if (ret) return ret; - } else - intel_state->cdclk = dev_priv->cdclk_freq; + } else { + intel_state->cdclk = dev_priv->atomic_cdclk_freq; + } ret = drm_atomic_helper_check_planes(dev, state); if (ret) @@ -14154,13 +14111,10 @@ static int intel_atomic_check(struct drm_device *dev, } static int intel_atomic_prepare_commit(struct drm_device *dev, - struct drm_atomic_state *state, - bool nonblock) + struct drm_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_plane_state *plane_state; struct drm_crtc_state *crtc_state; - struct drm_plane *plane; struct drm_crtc *crtc; int i, ret; @@ -14183,28 +14137,6 @@ static int intel_atomic_prepare_commit(struct drm_device *dev, ret = drm_atomic_helper_prepare_planes(dev, state); mutex_unlock(&dev->struct_mutex); - if (!ret && !nonblock) { - for_each_plane_in_state(state, plane, plane_state, i) { - struct intel_plane_state *intel_plane_state = - to_intel_plane_state(plane_state); - - if (!intel_plane_state->wait_req) - continue; - - ret = i915_wait_request(intel_plane_state->wait_req, - I915_WAIT_INTERRUPTIBLE, - NULL, NULL); - if (ret) { - /* Any hang should be swallowed by the wait */ - WARN_ON(ret == -EIO); - mutex_lock(&dev->struct_mutex); - drm_atomic_helper_cleanup_planes(dev, state); - mutex_unlock(&dev->struct_mutex); - break; - } - } - } - return ret; } @@ -14230,22 +14162,24 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, return; for_each_pipe(dev_priv, pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); if (!((1 << pipe) & crtc_mask)) continue; - ret = drm_crtc_vblank_get(crtc); + ret = drm_crtc_vblank_get(&crtc->base); if (WARN_ON(ret != 0)) { crtc_mask &= ~(1 << pipe); continue; } - last_vblank_count[pipe] = drm_crtc_vblank_count(crtc); + last_vblank_count[pipe] = drm_crtc_vblank_count(&crtc->base); } for_each_pipe(dev_priv, pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); long lret; if (!((1 << pipe) & crtc_mask)) @@ -14253,12 +14187,12 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, lret = wait_event_timeout(dev->vblank[pipe].queue, last_vblank_count[pipe] != - drm_crtc_vblank_count(crtc), + drm_crtc_vblank_count(&crtc->base), msecs_to_jiffies(50)); WARN(!lret, "pipe %c vblank wait timed out\n", pipe_name(pipe)); - drm_crtc_vblank_put(crtc); + drm_crtc_vblank_put(&crtc->base); } } @@ -14332,7 +14266,7 @@ static void intel_update_crtcs(struct drm_atomic_state *state, static void skl_update_crtcs(struct drm_atomic_state *state, unsigned int *crtc_vblank_mask) { - struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = to_i915(state->dev); struct intel_atomic_state *intel_state = to_intel_atomic_state(state); struct drm_crtc *crtc; struct intel_crtc *intel_crtc; @@ -14341,6 +14275,14 @@ static void skl_update_crtcs(struct drm_atomic_state *state, unsigned int updated = 0; bool progress; enum pipe pipe; + int i; + + const struct skl_ddb_entry *entries[I915_MAX_PIPES] = {}; + + for_each_crtc_in_state(state, crtc, old_crtc_state, i) + /* ignore allocations for crtc's that have been turned off. */ + if (crtc->state->active) + entries[i] = &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb; /* * Whenever the number of active pipes changes, we need to make sure we @@ -14349,7 +14291,6 @@ static void skl_update_crtcs(struct drm_atomic_state *state, * cause pipe underruns and other bad stuff. */ do { - int i; progress = false; for_each_crtc_in_state(state, crtc, old_crtc_state, i) { @@ -14360,12 +14301,14 @@ static void skl_update_crtcs(struct drm_atomic_state *state, cstate = to_intel_crtc_state(crtc->state); pipe = intel_crtc->pipe; - if (updated & cmask || !crtc->state->active) + if (updated & cmask || !cstate->base.active) continue; - if (skl_ddb_allocation_overlaps(state, intel_crtc)) + + if (skl_ddb_allocation_overlaps(entries, &cstate->wm.skl.ddb, i)) continue; updated |= cmask; + entries[i] = &cstate->wm.skl.ddb; /* * If this is an already active pipe, it's DDB changed, @@ -14374,7 +14317,7 @@ static void skl_update_crtcs(struct drm_atomic_state *state, * new ddb allocation to take effect. */ if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb, - &intel_crtc->hw_ddb) && + &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) && !crtc->state->active_changed && intel_state->wm_results.dirty_pipes != updated) vbl_wait = true; @@ -14383,7 +14326,7 @@ static void skl_update_crtcs(struct drm_atomic_state *state, crtc_vblank_mask); if (vbl_wait) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); progress = true; } @@ -14398,37 +14341,15 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) struct drm_crtc_state *old_crtc_state; struct drm_crtc *crtc; struct intel_crtc_state *intel_cstate; - struct drm_plane *plane; - struct drm_plane_state *plane_state; bool hw_check = intel_state->modeset; unsigned long put_domains[I915_MAX_PIPES] = {}; unsigned crtc_vblank_mask = 0; - int i, ret; - - for_each_plane_in_state(state, plane, plane_state, i) { - struct intel_plane_state *intel_plane_state = - to_intel_plane_state(plane->state); - - if (!intel_plane_state->wait_req) - continue; - - ret = i915_wait_request(intel_plane_state->wait_req, - 0, NULL, NULL); - /* EIO should be eaten, and we can't get interrupted in the - * worker, and blocking commits have waited already. */ - WARN_ON(ret); - } + int i; drm_atomic_helper_wait_for_dependencies(state); - if (intel_state->modeset) { - memcpy(dev_priv->min_pixclk, intel_state->min_pixclk, - sizeof(intel_state->min_pixclk)); - dev_priv->active_crtcs = intel_state->active_crtcs; - dev_priv->atomic_cdclk_freq = intel_state->cdclk; - + if (intel_state->modeset) intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET); - } for_each_crtc_in_state(state, crtc, old_crtc_state, i) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -14461,8 +14382,17 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) intel_check_cpu_fifo_underruns(dev_priv); intel_check_pch_fifo_underruns(dev_priv); - if (!crtc->state->active) - intel_update_watermarks(crtc); + if (!crtc->state->active) { + /* + * Make sure we don't call initial_watermarks + * for ILK-style watermark updates. + */ + if (dev_priv->display.atomic_update_watermarks) + dev_priv->display.initial_watermarks(intel_state, + to_intel_crtc_state(crtc->state)); + else + intel_update_watermarks(intel_crtc); + } } } @@ -14485,7 +14415,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) if (!intel_can_enable_sagv(state)) intel_disable_sagv(dev_priv); - intel_modeset_verify_disabled(dev); + intel_modeset_verify_disabled(dev, state); } /* Complete the events for pipes that have now been disabled */ @@ -14528,7 +14458,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) intel_cstate = to_intel_crtc_state(crtc->state); if (dev_priv->display.optimize_watermarks) - dev_priv->display.optimize_watermarks(intel_cstate); + dev_priv->display.optimize_watermarks(intel_state, + intel_cstate); } for_each_crtc_in_state(state, crtc, old_crtc_state, i) { @@ -14537,7 +14468,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) if (put_domains[i]) modeset_put_power_domains(dev_priv, put_domains[i]); - intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state); + intel_modeset_verify_crtc(crtc, state, old_crtc_state, crtc->state); } if (intel_state->modeset && intel_can_enable_sagv(state)) @@ -14572,12 +14503,33 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) static void intel_atomic_commit_work(struct work_struct *work) { - struct drm_atomic_state *state = container_of(work, - struct drm_atomic_state, - commit_work); + struct drm_atomic_state *state = + container_of(work, struct drm_atomic_state, commit_work); + intel_atomic_commit_tail(state); } +static int __i915_sw_fence_call +intel_atomic_commit_ready(struct i915_sw_fence *fence, + enum i915_sw_fence_notify notify) +{ + struct intel_atomic_state *state = + container_of(fence, struct intel_atomic_state, commit_ready); + + switch (notify) { + case FENCE_COMPLETE: + if (state->base.commit_work.func) + queue_work(system_unbound_wq, &state->base.commit_work); + break; + + case FENCE_FREE: + drm_atomic_state_put(&state->base); + break; + } + + return NOTIFY_DONE; +} + static void intel_atomic_track_fbs(struct drm_atomic_state *state) { struct drm_plane_state *old_plane_state; @@ -14599,10 +14551,6 @@ static void intel_atomic_track_fbs(struct drm_atomic_state *state) * This function commits a top-level state object that has been validated * with drm_atomic_helper_check(). * - * FIXME: Atomic modeset support for i915 is not yet complete. At the moment - * nonblocking commits are only safe for pure plane updates. Everything else - * should work though. - * * RETURNS * Zero for success or -errno. */ @@ -14614,34 +14562,42 @@ static int intel_atomic_commit(struct drm_device *dev, struct drm_i915_private *dev_priv = to_i915(dev); int ret = 0; - if (intel_state->modeset && nonblock) { - DRM_DEBUG_KMS("nonblocking commit for modeset not yet implemented.\n"); - return -EINVAL; - } - ret = drm_atomic_helper_setup_commit(state, nonblock); if (ret) return ret; - INIT_WORK(&state->commit_work, intel_atomic_commit_work); + drm_atomic_state_get(state); + i915_sw_fence_init(&intel_state->commit_ready, + intel_atomic_commit_ready); - ret = intel_atomic_prepare_commit(dev, state, nonblock); + ret = intel_atomic_prepare_commit(dev, state); if (ret) { DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret); + i915_sw_fence_commit(&intel_state->commit_ready); return ret; } drm_atomic_helper_swap_state(state, true); dev_priv->wm.distrust_bios_wm = false; - dev_priv->wm.skl_results = intel_state->wm_results; intel_shared_dpll_commit(state); intel_atomic_track_fbs(state); + if (intel_state->modeset) { + memcpy(dev_priv->min_pixclk, intel_state->min_pixclk, + sizeof(intel_state->min_pixclk)); + dev_priv->active_crtcs = intel_state->active_crtcs; + dev_priv->atomic_cdclk_freq = intel_state->cdclk; + } + drm_atomic_state_get(state); - if (nonblock) - queue_work(system_unbound_wq, &state->commit_work); - else + INIT_WORK(&state->commit_work, + nonblock ? intel_atomic_commit_work : NULL); + + i915_sw_fence_commit(&intel_state->commit_ready); + if (!nonblock) { + i915_sw_fence_wait(&intel_state->commit_ready); intel_atomic_commit_tail(state); + } return 0; } @@ -14753,20 +14709,21 @@ int intel_prepare_plane_fb(struct drm_plane *plane, struct drm_plane_state *new_state) { - struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_atomic_state *intel_state = + to_intel_atomic_state(new_state->state); + struct drm_i915_private *dev_priv = to_i915(plane->dev); struct drm_framebuffer *fb = new_state->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb); - struct reservation_object *resv; - int ret = 0; + int ret; if (!obj && !old_obj) return 0; if (old_obj) { struct drm_crtc_state *crtc_state = - drm_atomic_get_existing_crtc_state(new_state->state, plane->state->crtc); + drm_atomic_get_existing_crtc_state(new_state->state, + plane->state->crtc); /* Big Hammer, we also need to ensure that any pending * MI_WAIT_FOR_EVENT inside a user batch buffer on the @@ -14779,52 +14736,58 @@ intel_prepare_plane_fb(struct drm_plane *plane, * This should only fail upon a hung GPU, in which case we * can safely continue. */ - if (needs_modeset(crtc_state)) - ret = i915_gem_object_wait_rendering(old_obj, true); - if (ret) { - /* GPU hangs should have been swallowed by the wait */ - WARN_ON(ret == -EIO); - return ret; + if (needs_modeset(crtc_state)) { + ret = i915_sw_fence_await_reservation(&intel_state->commit_ready, + old_obj->resv, NULL, + false, 0, + GFP_KERNEL); + if (ret < 0) + return ret; } } + if (new_state->fence) { /* explicit fencing */ + ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready, + new_state->fence, + I915_FENCE_TIMEOUT, + GFP_KERNEL); + if (ret < 0) + return ret; + } + if (!obj) return 0; - /* For framebuffer backed by dmabuf, wait for fence */ - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - long lret; - - lret = reservation_object_wait_timeout_rcu(resv, false, true, - MAX_SCHEDULE_TIMEOUT); - if (lret == -ERESTARTSYS) - return lret; + if (!new_state->fence) { /* implicit fencing */ + ret = i915_sw_fence_await_reservation(&intel_state->commit_ready, + obj->resv, NULL, + false, I915_FENCE_TIMEOUT, + GFP_KERNEL); + if (ret < 0) + return ret; - WARN(lret < 0, "waiting returns %li\n", lret); + i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY); } if (plane->type == DRM_PLANE_TYPE_CURSOR && - INTEL_INFO(dev)->cursor_needs_physical) { + INTEL_INFO(dev_priv)->cursor_needs_physical) { int align = IS_I830(dev_priv) ? 16 * 1024 : 256; ret = i915_gem_object_attach_phys(obj, align); - if (ret) + if (ret) { DRM_DEBUG_KMS("failed to attach phys object\n"); + return ret; + } } else { struct i915_vma *vma; vma = intel_pin_and_fence_fb_obj(fb, new_state->rotation); - if (IS_ERR(vma)) - ret = PTR_ERR(vma); - } - - if (ret == 0) { - to_intel_plane_state(new_state)->wait_req = - i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); + if (IS_ERR(vma)) { + DRM_DEBUG_KMS("failed to pin object\n"); + return PTR_ERR(vma); + } } - return ret; + return 0; } /** @@ -14840,9 +14803,8 @@ void intel_cleanup_plane_fb(struct drm_plane *plane, struct drm_plane_state *old_state) { - struct drm_device *dev = plane->dev; + struct drm_i915_private *dev_priv = to_i915(plane->dev); struct intel_plane_state *old_intel_state; - struct intel_plane_state *intel_state = to_intel_plane_state(plane->state); struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb); struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb); @@ -14852,11 +14814,8 @@ intel_cleanup_plane_fb(struct drm_plane *plane, return; if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR || - !INTEL_INFO(dev)->cursor_needs_physical)) + !INTEL_INFO(dev_priv)->cursor_needs_physical)) intel_unpin_fb_obj(old_state->fb, old_state->rotation); - - i915_gem_request_assign(&intel_state->wait_req, NULL); - i915_gem_request_assign(&old_intel_state->wait_req, NULL); } int @@ -14933,30 +14892,32 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc, struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *intel_cstate = to_intel_crtc_state(crtc->state); - struct intel_crtc_state *old_intel_state = + struct intel_crtc_state *old_intel_cstate = to_intel_crtc_state(old_crtc_state); + struct intel_atomic_state *old_intel_state = + to_intel_atomic_state(old_crtc_state->state); bool modeset = needs_modeset(crtc->state); - enum pipe pipe = intel_crtc->pipe; /* Perform vblank evasion around commit operation */ intel_pipe_update_start(intel_crtc); if (modeset) - return; + goto out; if (crtc->state->color_mgmt_changed || to_intel_crtc_state(crtc->state)->update_pipe) { intel_color_set_csc(crtc->state); intel_color_load_luts(crtc->state); } - if (intel_cstate->update_pipe) { - intel_update_pipe_config(intel_crtc, old_intel_state); - } else if (INTEL_GEN(dev_priv) >= 9) { + if (intel_cstate->update_pipe) + intel_update_pipe_config(intel_crtc, old_intel_cstate); + else if (INTEL_GEN(dev_priv) >= 9) skl_detach_scalers(intel_crtc); - I915_WRITE(PIPE_WM_LINETIME(pipe), - intel_cstate->wm.skl.optimal.linetime); - } +out: + if (dev_priv->display.atomic_update_watermarks) + dev_priv->display.atomic_update_watermarks(old_intel_state, + intel_cstate); } static void intel_finish_crtc_commit(struct drm_crtc *crtc, @@ -14976,9 +14937,6 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc, */ void intel_plane_destroy(struct drm_plane *plane) { - if (!plane) - return; - drm_plane_cleanup(plane); kfree(to_intel_plane(plane)); } @@ -14992,13 +14950,11 @@ const struct drm_plane_funcs intel_plane_funcs = { .atomic_set_property = intel_plane_atomic_set_property, .atomic_duplicate_state = intel_plane_duplicate_state, .atomic_destroy_state = intel_plane_destroy_state, - }; -static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, - int pipe) +static struct intel_plane * +intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *primary = NULL; struct intel_plane_state *state = NULL; const uint32_t *intel_primary_formats; @@ -15007,28 +14963,38 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, int ret; primary = kzalloc(sizeof(*primary), GFP_KERNEL); - if (!primary) + if (!primary) { + ret = -ENOMEM; goto fail; + } state = intel_create_plane_state(&primary->base); - if (!state) + if (!state) { + ret = -ENOMEM; goto fail; + } + primary->base.state = &state->base; primary->can_scale = false; primary->max_downscale = 1; - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { primary->can_scale = true; state->scaler_id = -1; } primary->pipe = pipe; - primary->plane = pipe; + /* + * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS + * port is hooked to pipe B. Hence we want plane A feeding pipe B. + */ + if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4) + primary->plane = (enum plane) !pipe; + else + primary->plane = (enum plane) pipe; primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe); primary->check_plane = intel_check_primary_plane; - if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) - primary->plane = !pipe; - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { intel_primary_formats = skl_primary_formats; num_formats = ARRAY_SIZE(skl_primary_formats); @@ -15040,7 +15006,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, primary->update_plane = ironlake_update_primary_plane; primary->disable_plane = i9xx_disable_primary_plane; - } else if (INTEL_INFO(dev)->gen >= 4) { + } else if (INTEL_GEN(dev_priv) >= 4) { intel_primary_formats = i965_primary_formats; num_formats = ARRAY_SIZE(i965_primary_formats); @@ -15054,21 +15020,21 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, primary->disable_plane = i9xx_disable_primary_plane; } - if (INTEL_INFO(dev)->gen >= 9) - ret = drm_universal_plane_init(dev, &primary->base, 0, - &intel_plane_funcs, + if (INTEL_GEN(dev_priv) >= 9) + ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, + 0, &intel_plane_funcs, intel_primary_formats, num_formats, DRM_PLANE_TYPE_PRIMARY, "plane 1%c", pipe_name(pipe)); else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) - ret = drm_universal_plane_init(dev, &primary->base, 0, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, + 0, &intel_plane_funcs, intel_primary_formats, num_formats, DRM_PLANE_TYPE_PRIMARY, "primary %c", pipe_name(pipe)); else - ret = drm_universal_plane_init(dev, &primary->base, 0, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, + 0, &intel_plane_funcs, intel_primary_formats, num_formats, DRM_PLANE_TYPE_PRIMARY, "plane %c", plane_name(primary->plane)); @@ -15079,6 +15045,10 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, supported_rotations = DRM_ROTATE_0 | DRM_ROTATE_90 | DRM_ROTATE_180 | DRM_ROTATE_270; + } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { + supported_rotations = + DRM_ROTATE_0 | DRM_ROTATE_180 | + DRM_REFLECT_X; } else if (INTEL_GEN(dev_priv) >= 4) { supported_rotations = DRM_ROTATE_0 | DRM_ROTATE_180; @@ -15093,13 +15063,13 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs); - return &primary->base; + return primary; fail: kfree(state); kfree(primary); - return NULL; + return ERR_PTR(ret); } static int @@ -15139,7 +15109,7 @@ intel_check_cursor_plane(struct drm_plane *plane, return -ENOMEM; } - if (fb->modifier[0] != DRM_FORMAT_MOD_NONE) { + if (fb->modifier != DRM_FORMAT_MOD_NONE) { DRM_DEBUG_KMS("cursor cannot be tiled\n"); return -EINVAL; } @@ -15180,13 +15150,13 @@ intel_update_cursor_plane(struct drm_plane *plane, { struct drm_crtc *crtc = crtc_state->base.crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_device *dev = plane->dev; + struct drm_i915_private *dev_priv = to_i915(plane->dev); struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb); uint32_t addr; if (!obj) addr = 0; - else if (!INTEL_INFO(dev)->cursor_needs_physical) + else if (!INTEL_INFO(dev_priv)->cursor_needs_physical) addr = i915_gem_object_ggtt_offset(obj, NULL); else addr = obj->phys_handle->busaddr; @@ -15195,21 +15165,25 @@ intel_update_cursor_plane(struct drm_plane *plane, intel_crtc_update_cursor(crtc, state); } -static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, - int pipe) +static struct intel_plane * +intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *cursor = NULL; struct intel_plane_state *state = NULL; int ret; cursor = kzalloc(sizeof(*cursor), GFP_KERNEL); - if (!cursor) + if (!cursor) { + ret = -ENOMEM; goto fail; + } state = intel_create_plane_state(&cursor->base); - if (!state) + if (!state) { + ret = -ENOMEM; goto fail; + } + cursor->base.state = &state->base; cursor->can_scale = false; @@ -15221,8 +15195,8 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, cursor->update_plane = intel_update_cursor_plane; cursor->disable_plane = intel_disable_cursor_plane; - ret = drm_universal_plane_init(dev, &cursor->base, 0, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base, + 0, &intel_plane_funcs, intel_cursor_formats, ARRAY_SIZE(intel_cursor_formats), DRM_PLANE_TYPE_CURSOR, @@ -15236,90 +15210,100 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, DRM_ROTATE_0 | DRM_ROTATE_180); - if (INTEL_INFO(dev)->gen >=9) + if (INTEL_GEN(dev_priv) >= 9) state->scaler_id = -1; drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs); - return &cursor->base; + return cursor; fail: kfree(state); kfree(cursor); - return NULL; + return ERR_PTR(ret); } -static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc, - struct intel_crtc_state *crtc_state) +static void skl_init_scalers(struct drm_i915_private *dev_priv, + struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state) { + struct intel_crtc_scaler_state *scaler_state = + &crtc_state->scaler_state; int i; - struct intel_scaler *intel_scaler; - struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state; - for (i = 0; i < intel_crtc->num_scalers; i++) { - intel_scaler = &scaler_state->scalers[i]; - intel_scaler->in_use = 0; - intel_scaler->mode = PS_SCALER_MODE_DYN; + for (i = 0; i < crtc->num_scalers; i++) { + struct intel_scaler *scaler = &scaler_state->scalers[i]; + + scaler->in_use = 0; + scaler->mode = PS_SCALER_MODE_DYN; } scaler_state->scaler_id = -1; } -static void intel_crtc_init(struct drm_device *dev, int pipe) +static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc; struct intel_crtc_state *crtc_state = NULL; - struct drm_plane *primary = NULL; - struct drm_plane *cursor = NULL; - int ret; + struct intel_plane *primary = NULL; + struct intel_plane *cursor = NULL; + int sprite, ret; intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL); - if (intel_crtc == NULL) - return; + if (!intel_crtc) + return -ENOMEM; crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL); - if (!crtc_state) + if (!crtc_state) { + ret = -ENOMEM; goto fail; + } intel_crtc->config = crtc_state; intel_crtc->base.state = &crtc_state->base; crtc_state->base.crtc = &intel_crtc->base; /* initialize shared scalers */ - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { if (pipe == PIPE_C) intel_crtc->num_scalers = 1; else intel_crtc->num_scalers = SKL_NUM_SCALERS; - skl_init_scalers(dev, intel_crtc, crtc_state); + skl_init_scalers(dev_priv, intel_crtc, crtc_state); } - primary = intel_primary_plane_create(dev, pipe); - if (!primary) + primary = intel_primary_plane_create(dev_priv, pipe); + if (IS_ERR(primary)) { + ret = PTR_ERR(primary); goto fail; + } + + for_each_sprite(dev_priv, pipe, sprite) { + struct intel_plane *plane; + + plane = intel_sprite_plane_create(dev_priv, pipe, sprite); + if (IS_ERR(plane)) { + ret = PTR_ERR(plane); + goto fail; + } + } - cursor = intel_cursor_plane_create(dev, pipe); - if (!cursor) + cursor = intel_cursor_plane_create(dev_priv, pipe); + if (IS_ERR(cursor)) { + ret = PTR_ERR(cursor); goto fail; + } - ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary, - cursor, &intel_crtc_funcs, + ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base, + &primary->base, &cursor->base, + &intel_crtc_funcs, "pipe %c", pipe_name(pipe)); if (ret) goto fail; - /* - * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port - * is hooked to pipe B. Hence we want plane A feeding pipe B. - */ intel_crtc->pipe = pipe; - intel_crtc->plane = pipe; - if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) { - DRM_DEBUG_KMS("swapping pipes & planes for FBC\n"); - intel_crtc->plane = !pipe; - } + intel_crtc->plane = primary->plane; intel_crtc->cursor_base = ~0; intel_crtc->cursor_cntl = ~0; @@ -15329,21 +15313,26 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL); - dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base; - dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base; + dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = intel_crtc; + dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = intel_crtc; drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs); intel_color_init(&intel_crtc->base); WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe); - return; + + return 0; fail: - intel_plane_destroy(primary); - intel_plane_destroy(cursor); + /* + * drm_mode_config_cleanup() will free up any + * crtcs/planes already initialized. + */ kfree(crtc_state); kfree(intel_crtc); + + return ret; } enum pipe intel_get_pipe_from_connector(struct intel_connector *connector) @@ -15393,11 +15382,9 @@ static int intel_encoder_clones(struct intel_encoder *encoder) return index_mask; } -static bool has_edp_a(struct drm_device *dev) +static bool has_edp_a(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - if (!IS_MOBILE(dev)) + if (!IS_MOBILE(dev_priv)) return false; if ((I915_READ(DP_A) & DP_DETECTED) == 0) @@ -15409,11 +15396,9 @@ static bool has_edp_a(struct drm_device *dev) return true; } -static bool intel_crt_present(struct drm_device *dev) +static bool intel_crt_present(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) return false; if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) @@ -15487,7 +15472,7 @@ static void intel_setup_outputs(struct drm_device *dev) */ intel_lvds_init(dev); - if (intel_crt_present(dev)) + if (intel_crt_present(dev_priv)) intel_crt_init(dev); if (IS_BROXTON(dev_priv)) { @@ -15535,9 +15520,9 @@ static void intel_setup_outputs(struct drm_device *dev) } else if (HAS_PCH_SPLIT(dev_priv)) { int found; - dpd_is_edp = intel_dp_is_edp(dev, PORT_D); + dpd_is_edp = intel_dp_is_edp(dev_priv, PORT_D); - if (has_edp_a(dev)) + if (has_edp_a(dev_priv)) intel_dp_init(dev, DP_A, PORT_A); if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) { @@ -15578,14 +15563,14 @@ static void intel_setup_outputs(struct drm_device *dev) * trust the port type the VBT declares as we've seen at least * HDMI ports that the VBT claim are DP or eDP. */ - has_edp = intel_dp_is_edp(dev, PORT_B); + has_edp = intel_dp_is_edp(dev_priv, PORT_B); has_port = intel_bios_is_port_present(dev_priv, PORT_B); if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port) has_edp &= intel_dp_init(dev, VLV_DP_B, PORT_B); if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp) intel_hdmi_init(dev, VLV_HDMIB, PORT_B); - has_edp = intel_dp_is_edp(dev, PORT_C); + has_edp = intel_dp_is_edp(dev_priv, PORT_C); has_port = intel_bios_is_port_present(dev_priv, PORT_C); if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port) has_edp &= intel_dp_init(dev, VLV_DP_C, PORT_C); @@ -15642,7 +15627,7 @@ static void intel_setup_outputs(struct drm_device *dev) } else if (IS_GEN2(dev_priv)) intel_dvo_init(dev); - if (SUPPORTS_TV(dev)) + if (SUPPORTS_TV(dev_priv)) intel_tv_init(dev); intel_psr_init(dev); @@ -15697,6 +15682,8 @@ static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb, struct drm_i915_gem_object *obj = intel_fb->obj; mutex_lock(&dev->struct_mutex); + if (obj->pin_display && obj->cache_dirty) + i915_gem_clflush_object(obj, true); intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB); mutex_unlock(&dev->struct_mutex); @@ -15750,7 +15737,7 @@ static int intel_framebuffer_init(struct drm_device *dev, unsigned int tiling = i915_gem_object_get_tiling(obj); int ret; u32 pitch_limit, stride_alignment; - char *format_name; + struct drm_format_name_buf format_name; WARN_ON(!mutex_is_locked(&dev->struct_mutex)); @@ -15777,7 +15764,7 @@ static int intel_framebuffer_init(struct drm_device *dev, switch (mode_cmd->modifier[0]) { case I915_FORMAT_MOD_Y_TILED: case I915_FORMAT_MOD_Yf_TILED: - if (INTEL_INFO(dev)->gen < 9) { + if (INTEL_GEN(dev_priv) < 9) { DRM_DEBUG("Unsupported tiling 0x%llx!\n", mode_cmd->modifier[0]); return -EINVAL; @@ -15840,37 +15827,33 @@ static int intel_framebuffer_init(struct drm_device *dev, case DRM_FORMAT_ARGB8888: break; case DRM_FORMAT_XRGB1555: - if (INTEL_INFO(dev)->gen > 3) { - format_name = drm_get_format_name(mode_cmd->pixel_format); - DRM_DEBUG("unsupported pixel format: %s\n", format_name); - kfree(format_name); + if (INTEL_GEN(dev_priv) > 3) { + DRM_DEBUG("unsupported pixel format: %s\n", + drm_get_format_name(mode_cmd->pixel_format, &format_name)); return -EINVAL; } break; case DRM_FORMAT_ABGR8888: if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && - INTEL_INFO(dev)->gen < 9) { - format_name = drm_get_format_name(mode_cmd->pixel_format); - DRM_DEBUG("unsupported pixel format: %s\n", format_name); - kfree(format_name); + INTEL_GEN(dev_priv) < 9) { + DRM_DEBUG("unsupported pixel format: %s\n", + drm_get_format_name(mode_cmd->pixel_format, &format_name)); return -EINVAL; } break; case DRM_FORMAT_XBGR8888: case DRM_FORMAT_XRGB2101010: case DRM_FORMAT_XBGR2101010: - if (INTEL_INFO(dev)->gen < 4) { - format_name = drm_get_format_name(mode_cmd->pixel_format); - DRM_DEBUG("unsupported pixel format: %s\n", format_name); - kfree(format_name); + if (INTEL_GEN(dev_priv) < 4) { + DRM_DEBUG("unsupported pixel format: %s\n", + drm_get_format_name(mode_cmd->pixel_format, &format_name)); return -EINVAL; } break; case DRM_FORMAT_ABGR2101010: if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) { - format_name = drm_get_format_name(mode_cmd->pixel_format); - DRM_DEBUG("unsupported pixel format: %s\n", format_name); - kfree(format_name); + DRM_DEBUG("unsupported pixel format: %s\n", + drm_get_format_name(mode_cmd->pixel_format, &format_name)); return -EINVAL; } break; @@ -15878,17 +15861,15 @@ static int intel_framebuffer_init(struct drm_device *dev, case DRM_FORMAT_UYVY: case DRM_FORMAT_YVYU: case DRM_FORMAT_VYUY: - if (INTEL_INFO(dev)->gen < 5) { - format_name = drm_get_format_name(mode_cmd->pixel_format); - DRM_DEBUG("unsupported pixel format: %s\n", format_name); - kfree(format_name); + if (INTEL_GEN(dev_priv) < 5) { + DRM_DEBUG("unsupported pixel format: %s\n", + drm_get_format_name(mode_cmd->pixel_format, &format_name)); return -EINVAL; } break; default: - format_name = drm_get_format_name(mode_cmd->pixel_format); - DRM_DEBUG("unsupported pixel format: %s\n", format_name); - kfree(format_name); + DRM_DEBUG("unsupported pixel format: %s\n", + drm_get_format_name(mode_cmd->pixel_format, &format_name)); return -EINVAL; } @@ -15929,7 +15910,7 @@ intel_user_framebuffer_create(struct drm_device *dev, fb = intel_framebuffer_create(dev, &mode_cmd, obj); if (IS_ERR(fb)) - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return fb; } @@ -16309,9 +16290,8 @@ static void intel_init_quirks(struct drm_device *dev) } /* Disable the VGA plane that we never use */ -static void i915_disable_vga(struct drm_device *dev) +static void i915_disable_vga(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = dev_priv->drm.pdev; u8 sr1; i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv); @@ -16332,11 +16312,11 @@ void intel_modeset_init_hw(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq; - intel_init_clock_gating(dev); + intel_init_clock_gating(dev_priv); } /* @@ -16353,6 +16333,7 @@ static void sanitize_watermarks(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); struct drm_atomic_state *state; + struct intel_atomic_state *intel_state; struct drm_crtc *crtc; struct drm_crtc_state *cstate; struct drm_modeset_acquire_ctx ctx; @@ -16381,12 +16362,14 @@ retry: if (WARN_ON(IS_ERR(state))) goto fail; + intel_state = to_intel_atomic_state(state); + /* * Hardware readout is the only time we don't want to calculate * intermediate watermarks (since we don't trust the current * watermarks). */ - to_intel_atomic_state(state)->skip_intermediate_wm = true; + intel_state->skip_intermediate_wm = true; ret = intel_atomic_check(dev, state); if (ret) { @@ -16410,7 +16393,7 @@ retry: struct intel_crtc_state *cs = to_intel_crtc_state(cstate); cs->wm.need_postvbl_update = true; - dev_priv->display.optimize_watermarks(cs); + dev_priv->display.optimize_watermarks(intel_state, cs); } put_state: @@ -16420,11 +16403,10 @@ fail: drm_modeset_acquire_fini(&ctx); } -void intel_modeset_init(struct drm_device *dev) +int intel_modeset_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; - int sprite, ret; enum pipe pipe; struct intel_crtc *crtc; @@ -16442,10 +16424,10 @@ void intel_modeset_init(struct drm_device *dev) intel_init_quirks(dev); - intel_init_pm(dev); + intel_init_pm(dev_priv); - if (INTEL_INFO(dev)->num_pipes == 0) - return; + if (INTEL_INFO(dev_priv)->num_pipes == 0) + return 0; /* * There may be no VBT; and if the BIOS enabled SSC we can @@ -16490,29 +16472,30 @@ void intel_modeset_init(struct drm_device *dev) dev->mode_config.fb_base = ggtt->mappable_base; DRM_DEBUG_KMS("%d display pipe%s available.\n", - INTEL_INFO(dev)->num_pipes, - INTEL_INFO(dev)->num_pipes > 1 ? "s" : ""); + INTEL_INFO(dev_priv)->num_pipes, + INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : ""); for_each_pipe(dev_priv, pipe) { - intel_crtc_init(dev, pipe); - for_each_sprite(dev_priv, pipe, sprite) { - ret = intel_plane_init(dev, pipe, sprite); - if (ret) - DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n", - pipe_name(pipe), sprite_name(pipe, sprite), ret); + int ret; + + ret = intel_crtc_init(dev_priv, pipe); + if (ret) { + drm_mode_config_cleanup(dev); + return ret; } } intel_update_czclk(dev_priv); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); + dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq; intel_shared_dpll_init(dev); if (dev_priv->max_cdclk_freq == 0) - intel_update_max_cdclk(dev); + intel_update_max_cdclk(dev_priv); /* Just disable it once at startup */ - i915_disable_vga(dev); + i915_disable_vga(dev_priv); intel_setup_outputs(dev); drm_modeset_lock_all(dev); @@ -16548,6 +16531,8 @@ void intel_modeset_init(struct drm_device *dev) * since the watermark calculation done here will use pstate->fb. */ sanitize_watermarks(dev); + + return 0; } static void intel_enable_pipe_a(struct drm_device *dev) @@ -16577,11 +16562,10 @@ static void intel_enable_pipe_a(struct drm_device *dev) static bool intel_check_plane_mapping(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 val; - if (INTEL_INFO(dev)->num_pipes == 1) + if (INTEL_INFO(dev_priv)->num_pipes == 1) return true; val = I915_READ(DSPCNTR(!crtc->plane)); @@ -16655,7 +16639,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) /* We need to sanitize the plane -> pipe mapping first because this will * disable the crtc (and hence change the state) if it is wrong. Note * that gen4+ has a fixed plane -> pipe mapping. */ - if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) { + if (INTEL_GEN(dev_priv) < 4 && !intel_check_plane_mapping(crtc)) { bool plane; DRM_DEBUG_KMS("[CRTC:%d:%s] wrong plane connection detected!\n", @@ -16757,21 +16741,18 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) * the crtc fixup. */ } -void i915_redisable_vga_power_on(struct drm_device *dev) +void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv); if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) { DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n"); - i915_disable_vga(dev); + i915_disable_vga(dev_priv); } } -void i915_redisable_vga(struct drm_device *dev) +void i915_redisable_vga(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* This function can be called both from intel_modeset_setup_hw_state or * at a very early point in our resume sequence, where the power well * structures are not yet restored. Since this function is at a very @@ -16782,7 +16763,7 @@ void i915_redisable_vga(struct drm_device *dev) if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA)) return; - i915_redisable_vga_power_on(dev); + i915_redisable_vga_power_on(dev_priv); intel_display_power_put(dev_priv, POWER_DOMAIN_VGA); } @@ -16854,7 +16835,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n", crtc->base.base.id, crtc->base.name, - crtc->active ? "enabled" : "disabled"); + enableddisabled(crtc->active)); } for (i = 0; i < dev_priv->num_shared_dpll; i++) { @@ -16877,7 +16858,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) pipe = 0; if (encoder->get_hw_state(encoder, &pipe)) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + encoder->base.crtc = &crtc->base; crtc->config->output_types |= 1 << encoder->type; encoder->get_config(encoder, crtc->config); @@ -16886,9 +16868,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) } DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n", - encoder->base.base.id, - encoder->base.name, - encoder->base.crtc ? "enabled" : "disabled", + encoder->base.base.id, encoder->base.name, + enableddisabled(encoder->base.crtc), pipe_name(pipe)); } @@ -16917,9 +16898,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) connector->base.encoder = NULL; } DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n", - connector->base.base.id, - connector->base.name, - connector->base.encoder ? "enabled" : "disabled"); + connector->base.base.id, connector->base.name, + enableddisabled(connector->base.encoder)); } for_each_intel_crtc(dev, crtc) { @@ -16978,7 +16958,8 @@ intel_modeset_setup_hw_state(struct drm_device *dev) } for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + intel_sanitize_crtc(crtc); intel_dump_pipe_config(crtc, crtc->config, "[setup_hw_state]"); @@ -17166,10 +17147,9 @@ void intel_connector_attach_encoder(struct intel_connector *connector, /* * set vga decode state - true == enable VGA decode */ -int intel_modeset_vga_set_state(struct drm_device *dev, bool state) +int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state) { - struct drm_i915_private *dev_priv = to_i915(dev); - unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL; + unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL; u16 gmch_ctrl; if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) { @@ -17323,16 +17303,15 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv) void intel_display_print_error_state(struct drm_i915_error_state_buf *m, - struct drm_device *dev, + struct drm_i915_private *dev_priv, struct intel_display_error_state *error) { - struct drm_i915_private *dev_priv = to_i915(dev); int i; if (!error) return; - err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes); + err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes); if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) err_printf(m, "PWR_WELL_CTL2: %08x\n", error->power_well_driver); @@ -17346,13 +17325,13 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m, err_printf(m, "Plane [%d]:\n", i); err_printf(m, " CNTR: %08x\n", error->plane[i].control); err_printf(m, " STRIDE: %08x\n", error->plane[i].stride); - if (INTEL_INFO(dev)->gen <= 3) { + if (INTEL_GEN(dev_priv) <= 3) { err_printf(m, " SIZE: %08x\n", error->plane[i].size); err_printf(m, " POS: %08x\n", error->plane[i].pos); } if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv)) err_printf(m, " ADDR: %08x\n", error->plane[i].addr); - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { err_printf(m, " SURF: %08x\n", error->plane[i].surface); err_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset); } diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 3c2293bd24bf..90283edcafba 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -213,6 +213,81 @@ intel_dp_downstream_max_dotclock(struct intel_dp *intel_dp) return max_dotclk; } +static int +intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates) +{ + if (intel_dp->num_sink_rates) { + *sink_rates = intel_dp->sink_rates; + return intel_dp->num_sink_rates; + } + + *sink_rates = default_rates; + + return (intel_dp_max_link_bw(intel_dp) >> 3) + 1; +} + +static int +intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); + int size; + + if (IS_BROXTON(dev_priv)) { + *source_rates = bxt_rates; + size = ARRAY_SIZE(bxt_rates); + } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + *source_rates = skl_rates; + size = ARRAY_SIZE(skl_rates); + } else { + *source_rates = default_rates; + size = ARRAY_SIZE(default_rates); + } + + /* This depends on the fact that 5.4 is last value in the array */ + if (!intel_dp_source_supports_hbr2(intel_dp)) + size--; + + return size; +} + +static int intersect_rates(const int *source_rates, int source_len, + const int *sink_rates, int sink_len, + int *common_rates) +{ + int i = 0, j = 0, k = 0; + + while (i < source_len && j < sink_len) { + if (source_rates[i] == sink_rates[j]) { + if (WARN_ON(k >= DP_MAX_SUPPORTED_RATES)) + return k; + common_rates[k] = source_rates[i]; + ++k; + ++i; + ++j; + } else if (source_rates[i] < sink_rates[j]) { + ++i; + } else { + ++j; + } + } + return k; +} + +static int intel_dp_common_rates(struct intel_dp *intel_dp, + int *common_rates) +{ + const int *source_rates, *sink_rates; + int source_len, sink_len; + + sink_len = intel_dp_sink_rates(intel_dp, &sink_rates); + source_len = intel_dp_source_rates(intel_dp, &source_rates); + + return intersect_rates(source_rates, source_len, + sink_rates, sink_len, + common_rates); +} + static enum drm_mode_status intel_dp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) @@ -320,8 +395,7 @@ static void vlv_power_sequencer_kick(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); enum pipe pipe = intel_dp->pps_pipe; bool pll_enabled, release_cl_override = false; enum dpio_phy phy = DPIO_PHY(pipe); @@ -359,7 +433,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) release_cl_override = IS_CHERRYVIEW(dev_priv) && !chv_phy_powergate_ch(dev_priv, phy, ch, true); - if (vlv_force_pll_on(dev, pipe, IS_CHERRYVIEW(dev_priv) ? + if (vlv_force_pll_on(dev_priv, pipe, IS_CHERRYVIEW(dev_priv) ? &chv_dpll[0].dpll : &vlv_dpll[0].dpll)) { DRM_ERROR("Failed to force on pll for pipe %c!\n", pipe_name(pipe)); @@ -383,7 +457,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) POSTING_READ(intel_dp->output_reg); if (!pll_enabled) { - vlv_force_pll_off(dev, pipe); + vlv_force_pll_off(dev_priv, pipe); if (release_cl_override) chv_phy_powergate_ch(dev_priv, phy, ch, false); @@ -868,14 +942,14 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, uint8_t *recv, int recv_size) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = + to_i915(intel_dig_port->base.base.dev); i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; uint32_t aux_clock_divider; int i, ret, recv_bytes; uint32_t status; int try, clock = 0; - bool has_aux_irq = HAS_AUX_IRQ(dev); + bool has_aux_irq = HAS_AUX_IRQ(dev_priv); bool vdd; pps_lock(intel_dp); @@ -1291,19 +1365,6 @@ intel_dp_aux_init(struct intel_dp *intel_dp) intel_dp->aux.transfer = intel_dp_aux_transfer; } -static int -intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates) -{ - if (intel_dp->num_sink_rates) { - *sink_rates = intel_dp->sink_rates; - return intel_dp->num_sink_rates; - } - - *sink_rates = default_rates; - - return (intel_dp_max_link_bw(intel_dp) >> 3) + 1; -} - bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); @@ -1316,31 +1377,6 @@ bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp) return false; } -static int -intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates) -{ - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - int size; - - if (IS_BROXTON(dev_priv)) { - *source_rates = bxt_rates; - size = ARRAY_SIZE(bxt_rates); - } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { - *source_rates = skl_rates; - size = ARRAY_SIZE(skl_rates); - } else { - *source_rates = default_rates; - size = ARRAY_SIZE(default_rates); - } - - /* This depends on the fact that 5.4 is last value in the array */ - if (!intel_dp_source_supports_hbr2(intel_dp)) - size--; - - return size; -} - static void intel_dp_set_clock(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) @@ -1375,43 +1411,6 @@ intel_dp_set_clock(struct intel_encoder *encoder, } } -static int intersect_rates(const int *source_rates, int source_len, - const int *sink_rates, int sink_len, - int *common_rates) -{ - int i = 0, j = 0, k = 0; - - while (i < source_len && j < sink_len) { - if (source_rates[i] == sink_rates[j]) { - if (WARN_ON(k >= DP_MAX_SUPPORTED_RATES)) - return k; - common_rates[k] = source_rates[i]; - ++k; - ++i; - ++j; - } else if (source_rates[i] < sink_rates[j]) { - ++i; - } else { - ++j; - } - } - return k; -} - -static int intel_dp_common_rates(struct intel_dp *intel_dp, - int *common_rates) -{ - const int *source_rates, *sink_rates; - int source_len, sink_len; - - sink_len = intel_dp_sink_rates(intel_dp, &sink_rates); - source_len = intel_dp_source_rates(intel_dp, &source_rates); - - return intersect_rates(source_rates, source_len, - sink_rates, sink_len, - common_rates); -} - static void snprintf_int_array(char *str, size_t len, const int *array, int nelem) { @@ -1451,40 +1450,35 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp) DRM_DEBUG_KMS("common rates: %s\n", str); } -static void intel_dp_print_hw_revision(struct intel_dp *intel_dp) +bool +__intel_dp_read_desc(struct intel_dp *intel_dp, struct intel_dp_desc *desc) { - uint8_t rev; - int len; + u32 base = drm_dp_is_branch(intel_dp->dpcd) ? DP_BRANCH_OUI : + DP_SINK_OUI; - if ((drm_debug & DRM_UT_KMS) == 0) - return; - - if (!drm_dp_is_branch(intel_dp->dpcd)) - return; - - len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_HW_REV, &rev, 1); - if (len < 0) - return; - - DRM_DEBUG_KMS("sink hw revision: %d.%d\n", (rev & 0xf0) >> 4, rev & 0xf); + return drm_dp_dpcd_read(&intel_dp->aux, base, desc, sizeof(*desc)) == + sizeof(*desc); } -static void intel_dp_print_sw_revision(struct intel_dp *intel_dp) +bool intel_dp_read_desc(struct intel_dp *intel_dp) { - uint8_t rev[2]; - int len; - - if ((drm_debug & DRM_UT_KMS) == 0) - return; + struct intel_dp_desc *desc = &intel_dp->desc; + bool oui_sup = intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & + DP_OUI_SUPPORT; + int dev_id_len; - if (!drm_dp_is_branch(intel_dp->dpcd)) - return; + if (!__intel_dp_read_desc(intel_dp, desc)) + return false; - len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_SW_REV, &rev, 2); - if (len < 0) - return; + dev_id_len = strnlen(desc->device_id, sizeof(desc->device_id)); + DRM_DEBUG_KMS("DP %s: OUI %*phD%s dev-ID %*pE HW-rev %d.%d SW-rev %d.%d\n", + drm_dp_is_branch(intel_dp->dpcd) ? "branch" : "sink", + (int)sizeof(desc->oui), desc->oui, oui_sup ? "" : "(NS)", + dev_id_len, desc->device_id, + desc->hw_rev >> 4, desc->hw_rev & 0xf, + desc->sw_major_rev, desc->sw_minor_rev); - DRM_DEBUG_KMS("sink sw revision: %d.%d\n", rev[0], rev[1]); + return true; } static int rate_to_index(int find, const int *rates) @@ -1548,8 +1542,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = dp_to_dig_port(intel_dp)->port; @@ -1584,7 +1577,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, intel_fixed_panel_mode(intel_connector->panel.fixed_mode, adjusted_mode); - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { int ret; ret = skl_update_scaler_crtc(pipe_config); if (ret) @@ -1797,9 +1790,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder, trans_dp &= ~TRANS_DP_ENH_FRAMING; I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp); } else { - if (!HAS_PCH_SPLIT(dev_priv) && !IS_VALLEYVIEW(dev_priv) && - !IS_CHERRYVIEW(dev_priv) && - pipe_config->limited_color_range) + if (IS_G4X(dev_priv) && pipe_config->limited_color_range) intel_dp->DP |= DP_COLOR_RANGE_16_235; if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) @@ -2369,7 +2360,7 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp, * 2. Program DP PLL enable */ if (IS_GEN5(dev_priv)) - intel_wait_for_vblank_if_active(&dev_priv->drm, !crtc->pipe); + intel_wait_for_vblank_if_active(dev_priv, !crtc->pipe); intel_dp->DP |= DP_PLL_ENABLE; @@ -2521,8 +2512,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder, pipe_config->base.adjusted_mode.flags |= flags; - if (!HAS_PCH_SPLIT(dev_priv) && !IS_VALLEYVIEW(dev_priv) && - !IS_CHERRYVIEW(dev_priv) && tmp & DP_COLOR_RANGE_16_235) + if (IS_G4X(dev_priv) && tmp & DP_COLOR_RANGE_16_235) pipe_config->limited_color_range = true; pipe_config->lane_count = @@ -2741,7 +2731,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp, } static void intel_enable_dp(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct drm_device *dev = encoder->base.dev; @@ -2783,7 +2774,7 @@ static void intel_enable_dp(struct intel_encoder *encoder, if (pipe_config->has_audio) { DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n", pipe_name(pipe)); - intel_audio_codec_enable(encoder); + intel_audio_codec_enable(encoder, pipe_config, conn_state); } } @@ -2793,7 +2784,7 @@ static void g4x_enable_dp(struct intel_encoder *encoder, { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - intel_enable_dp(encoder, pipe_config); + intel_enable_dp(encoder, pipe_config, conn_state); intel_edp_backlight_on(intel_dp); } @@ -2930,7 +2921,7 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder, { vlv_phy_pre_encoder_enable(encoder); - intel_enable_dp(encoder, pipe_config); + intel_enable_dp(encoder, pipe_config, conn_state); } static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder, @@ -2948,7 +2939,7 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder, { chv_phy_pre_encoder_enable(encoder); - intel_enable_dp(encoder, pipe_config); + intel_enable_dp(encoder, pipe_config, conn_state); /* Second common lane will stay alive on its own now */ chv_phy_release_cl2_override(encoder); @@ -2985,13 +2976,12 @@ intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_ uint8_t intel_dp_voltage_max(struct intel_dp *intel_dp) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp)); enum port port = dp_to_dig_port(intel_dp)->port; if (IS_BROXTON(dev_priv)) return DP_TRAIN_VOLTAGE_SWING_LEVEL_3; - else if (INTEL_INFO(dev)->gen >= 9) { + else if (INTEL_GEN(dev_priv) >= 9) { if (dev_priv->vbt.edp.low_vswing && port == PORT_A) return DP_TRAIN_VOLTAGE_SWING_LEVEL_3; return DP_TRAIN_VOLTAGE_SWING_LEVEL_2; @@ -3492,7 +3482,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) I915_WRITE(intel_dp->output_reg, DP); POSTING_READ(intel_dp->output_reg); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -3502,7 +3492,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) intel_dp->DP = DP; } -static bool +bool intel_dp_read_dpcd(struct intel_dp *intel_dp) { if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd, @@ -3526,6 +3516,8 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) if (!intel_dp_read_dpcd(intel_dp)) return false; + intel_dp_read_desc(intel_dp); + if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] & DP_NO_AUX_HANDSHAKE_LINK_TRAINING; @@ -3627,23 +3619,6 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) return true; } -static void -intel_dp_probe_oui(struct intel_dp *intel_dp) -{ - u8 buf[3]; - - if (!(intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT)) - return; - - if (drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_OUI, buf, 3) == 3) - DRM_DEBUG_KMS("Sink OUI: %02hx%02hx%02hx\n", - buf[0], buf[1], buf[2]); - - if (drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_OUI, buf, 3) == 3) - DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n", - buf[0], buf[1], buf[2]); -} - static bool intel_dp_can_mst(struct intel_dp *intel_dp) { @@ -3687,7 +3662,7 @@ intel_dp_configure_mst(struct intel_dp *intel_dp) static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int ret = 0; @@ -3708,7 +3683,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) } do { - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { @@ -3731,7 +3706,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) static int intel_dp_sink_crc_start(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int ret; @@ -3759,14 +3734,14 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp) return -EIO; } - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); return 0; } int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int count, ret; @@ -3777,7 +3752,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) return ret; do { - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { @@ -4010,7 +3985,7 @@ intel_dp_retrain_link(struct intel_dp *intel_dp) intel_dp_stop_link_train(intel_dp); /* Keep underrun reporting disabled until things are stable */ - intel_wait_for_vblank(&dev_priv->drm, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); if (crtc->config->has_pch_encoder) @@ -4422,10 +4397,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector) intel_dp_print_rates(intel_dp); - intel_dp_probe_oui(intel_dp); - - intel_dp_print_hw_revision(intel_dp); - intel_dp_print_sw_revision(intel_dp); + intel_dp_read_desc(intel_dp); intel_dp_configure_mst(intel_dp); @@ -4489,21 +4461,11 @@ static enum drm_connector_status intel_dp_detect(struct drm_connector *connector, bool force) { struct intel_dp *intel_dp = intel_attached_dp(connector); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *intel_encoder = &intel_dig_port->base; enum drm_connector_status status = connector->status; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); - if (intel_dp->is_mst) { - /* MST devices are disconnected from a monitor POV */ - intel_dp_unset_edid(intel_dp); - if (intel_encoder->type != INTEL_OUTPUT_EDP) - intel_encoder->type = INTEL_OUTPUT_DP; - return connector_status_disconnected; - } - /* If full detect is not performed yet, do a full detect */ if (!intel_dp->detect_done) status = intel_dp_long_pulse(intel_dp->attached_connector); @@ -4907,15 +4869,13 @@ put_power: } /* check the VBT to see whether the eDP is on another port */ -bool intel_dp_is_edp(struct drm_device *dev, enum port port) +bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* * eDP not supported on g4x. so bail out early just * for a bit extra safety in case the VBT is bonkers. */ - if (INTEL_INFO(dev)->gen < 5) + if (INTEL_GEN(dev_priv) < 5) return false; if (port == PORT_A) @@ -5517,7 +5477,7 @@ intel_dp_drrs_init(struct intel_connector *intel_connector, INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work); mutex_init(&dev_priv->drrs.mutex); - if (INTEL_INFO(dev)->gen <= 6) { + if (INTEL_GEN(dev_priv) <= 6) { DRM_DEBUG_KMS("DRRS supported for Gen7 and above\n"); return NULL; } @@ -5691,7 +5651,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, intel_dp->pps_pipe = INVALID_PIPE; /* intel_dp vfuncs */ - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider; else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider; @@ -5700,7 +5660,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, else intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider; - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl; else intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl; @@ -5712,7 +5672,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, intel_dp->DP = I915_READ(intel_dp->output_reg); intel_dp->attached_connector = intel_connector; - if (intel_dp_is_edp(dev, port)) + if (intel_dp_is_edp(dev_priv, port)) type = DRM_MODE_CONNECTOR_eDP; else type = DRM_MODE_CONNECTOR_DisplayPort; @@ -5776,7 +5736,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, } /* init MST on ports that can support it */ - if (HAS_DP_MST(dev) && !is_edp(intel_dp) && + if (HAS_DP_MST(dev_priv) && !is_edp(intel_dp) && (port == PORT_B || port == PORT_C || port == PORT_D)) intel_dp_mst_encoder_init(intel_dig_port, intel_connector->base.base.id); @@ -5850,7 +5810,7 @@ bool intel_dp_init(struct drm_device *dev, } else { intel_encoder->pre_enable = g4x_pre_enable_dp; intel_encoder->enable = g4x_enable_dp; - if (INTEL_INFO(dev)->gen >= 5) + if (INTEL_GEN(dev_priv) >= 5) intel_encoder->post_disable = ilk_post_disable_dp; } diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c index 3ffbd69e4551..b029d1026a28 100644 --- a/drivers/gpu/drm/i915/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/intel_dp_mst.c @@ -43,7 +43,6 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; int mst_pbn; - pipe_config->dp_encoder_is_mst = true; pipe_config->has_pch_encoder = false; bpp = 24; /* diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c index 047f48748944..7a8e82dabbf2 100644 --- a/drivers/gpu/drm/i915/intel_dpio_phy.c +++ b/drivers/gpu/drm/i915/intel_dpio_phy.c @@ -23,6 +23,565 @@ #include "intel_drv.h" +/** + * DOC: DPIO + * + * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI + * ports. DPIO is the name given to such a display PHY. These PHYs + * don't follow the standard programming model using direct MMIO + * registers, and instead their registers must be accessed trough IOSF + * sideband. VLV has one such PHY for driving ports B and C, and CHV + * adds another PHY for driving port D. Each PHY responds to specific + * IOSF-SB port. + * + * Each display PHY is made up of one or two channels. Each channel + * houses a common lane part which contains the PLL and other common + * logic. CH0 common lane also contains the IOSF-SB logic for the + * Common Register Interface (CRI) ie. the DPIO registers. CRI clock + * must be running when any DPIO registers are accessed. + * + * In addition to having their own registers, the PHYs are also + * controlled through some dedicated signals from the display + * controller. These include PLL reference clock enable, PLL enable, + * and CRI clock selection, for example. + * + * Eeach channel also has two splines (also called data lanes), and + * each spline is made up of one Physical Access Coding Sub-Layer + * (PCS) block and two TX lanes. So each channel has two PCS blocks + * and four TX lanes. The TX lanes are used as DP lanes or TMDS + * data/clock pairs depending on the output type. + * + * Additionally the PHY also contains an AUX lane with AUX blocks + * for each channel. This is used for DP AUX communication, but + * this fact isn't really relevant for the driver since AUX is + * controlled from the display controller side. No DPIO registers + * need to be accessed during AUX communication, + * + * Generally on VLV/CHV the common lane corresponds to the pipe and + * the spline (PCS/TX) corresponds to the port. + * + * For dual channel PHY (VLV/CHV): + * + * pipe A == CMN/PLL/REF CH0 + * + * pipe B == CMN/PLL/REF CH1 + * + * port B == PCS/TX CH0 + * + * port C == PCS/TX CH1 + * + * This is especially important when we cross the streams + * ie. drive port B with pipe B, or port C with pipe A. + * + * For single channel PHY (CHV): + * + * pipe C == CMN/PLL/REF CH0 + * + * port D == PCS/TX CH0 + * + * On BXT the entire PHY channel corresponds to the port. That means + * the PLL is also now associated with the port rather than the pipe, + * and so the clock needs to be routed to the appropriate transcoder. + * Port A PLL is directly connected to transcoder EDP and port B/C + * PLLs can be routed to any transcoder A/B/C. + * + * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is + * digital port D (CHV) or port A (BXT). :: + * + * + * Dual channel PHY (VLV/CHV/BXT) + * --------------------------------- + * | CH0 | CH1 | + * | CMN/PLL/REF | CMN/PLL/REF | + * |---------------|---------------| Display PHY + * | PCS01 | PCS23 | PCS01 | PCS23 | + * |-------|-------|-------|-------| + * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| + * --------------------------------- + * | DDI0 | DDI1 | DP/HDMI ports + * --------------------------------- + * + * Single channel PHY (CHV/BXT) + * ----------------- + * | CH0 | + * | CMN/PLL/REF | + * |---------------| Display PHY + * | PCS01 | PCS23 | + * |-------|-------| + * |TX0|TX1|TX2|TX3| + * ----------------- + * | DDI2 | DP/HDMI port + * ----------------- + */ + +/** + * struct bxt_ddi_phy_info - Hold info for a broxton DDI phy + */ +struct bxt_ddi_phy_info { + /** + * @dual_channel: true if this phy has a second channel. + */ + bool dual_channel; + + /** + * @rcomp_phy: If -1, indicates this phy has its own rcomp resistor. + * Otherwise the GRC value will be copied from the phy indicated by + * this field. + */ + enum dpio_phy rcomp_phy; + + /** + * @channel: struct containing per channel information. + */ + struct { + /** + * @port: which port maps to this channel. + */ + enum port port; + } channel[2]; +}; + +static const struct bxt_ddi_phy_info bxt_ddi_phy_info[] = { + [DPIO_PHY0] = { + .dual_channel = true, + .rcomp_phy = DPIO_PHY1, + + .channel = { + [DPIO_CH0] = { .port = PORT_B }, + [DPIO_CH1] = { .port = PORT_C }, + } + }, + [DPIO_PHY1] = { + .dual_channel = false, + .rcomp_phy = -1, + + .channel = { + [DPIO_CH0] = { .port = PORT_A }, + } + }, +}; + +static u32 bxt_phy_port_mask(const struct bxt_ddi_phy_info *phy_info) +{ + return (phy_info->dual_channel * BIT(phy_info->channel[DPIO_CH1].port)) | + BIT(phy_info->channel[DPIO_CH0].port); +} + +void bxt_port_to_phy_channel(enum port port, + enum dpio_phy *phy, enum dpio_channel *ch) +{ + const struct bxt_ddi_phy_info *phy_info; + int i; + + for (i = 0; i < ARRAY_SIZE(bxt_ddi_phy_info); i++) { + phy_info = &bxt_ddi_phy_info[i]; + + if (port == phy_info->channel[DPIO_CH0].port) { + *phy = i; + *ch = DPIO_CH0; + return; + } + + if (phy_info->dual_channel && + port == phy_info->channel[DPIO_CH1].port) { + *phy = i; + *ch = DPIO_CH1; + return; + } + } + + WARN(1, "PHY not found for PORT %c", port_name(port)); + *phy = DPIO_PHY0; + *ch = DPIO_CH0; +} + +void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, + enum port port, u32 margin, u32 scale, + u32 enable, u32 deemphasis) +{ + u32 val; + enum dpio_phy phy; + enum dpio_channel ch; + + bxt_port_to_phy_channel(port, &phy, &ch); + + /* + * While we write to the group register to program all lanes at once we + * can read only lane registers and we pick lanes 0/1 for that. + */ + val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); + val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT); + I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_TX_DW2_LN0(phy, ch)); + val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE); + val |= margin << MARGIN_000_SHIFT | scale << UNIQ_TRANS_SCALE_SHIFT; + I915_WRITE(BXT_PORT_TX_DW2_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_TX_DW3_LN0(phy, ch)); + val &= ~SCALE_DCOMP_METHOD; + if (enable) + val |= SCALE_DCOMP_METHOD; + + if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) + DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set"); + + I915_WRITE(BXT_PORT_TX_DW3_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_TX_DW4_LN0(phy, ch)); + val &= ~DE_EMPHASIS; + val |= deemphasis << DEEMPH_SHIFT; + I915_WRITE(BXT_PORT_TX_DW4_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); + val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT; + I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); +} + +bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + enum port port; + + if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy))) + return false; + + if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & + (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n", + phy); + + return false; + } + + if (phy_info->rcomp_phy == -1 && + !(I915_READ(BXT_PORT_REF_DW3(phy)) & GRC_DONE)) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but GRC isn't done\n", + phy); + + return false; + } + + if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n", + phy); + + return false; + } + + for_each_port_masked(port, bxt_phy_port_mask(phy_info)) { + u32 tmp = I915_READ(BXT_PHY_CTL(port)); + + if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane " + "for port %c powered down " + "(PHY_CTL %08x)\n", + phy, port_name(port), tmp); + + return false; + } + } + + return true; +} + +static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) +{ + u32 val = I915_READ(BXT_PORT_REF_DW6(phy)); + + return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT; +} + +static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + if (intel_wait_for_register(dev_priv, + BXT_PORT_REF_DW3(phy), + GRC_DONE, GRC_DONE, + 10)) + DRM_ERROR("timeout waiting for PHY%d GRC\n", phy); +} + +static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + u32 val; + + if (bxt_ddi_phy_is_enabled(dev_priv, phy)) { + /* Still read out the GRC value for state verification */ + if (phy_info->rcomp_phy != -1) + dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy); + + if (bxt_ddi_phy_verify_state(dev_priv, phy)) { + DRM_DEBUG_DRIVER("DDI PHY %d already enabled, " + "won't reprogram it\n", phy); + + return; + } + + DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, " + "force reprogramming it\n", phy); + } + + val = I915_READ(BXT_P_CR_GT_DISP_PWRON); + val |= GT_DISPLAY_POWER_ON(phy); + I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); + + /* + * The PHY registers start out inaccessible and respond to reads with + * all 1s. Eventually they become accessible as they power up, then + * the reserved bit will give the default 0. Poll on the reserved bit + * becoming 0 to find when the PHY is accessible. + * HW team confirmed that the time to reach phypowergood status is + * anywhere between 50 us and 100us. + */ + if (wait_for_us(((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & + (PHY_RESERVED | PHY_POWER_GOOD)) == PHY_POWER_GOOD), 100)) { + DRM_ERROR("timeout during PHY%d power on\n", phy); + } + + /* Program PLL Rcomp code offset */ + val = I915_READ(BXT_PORT_CL1CM_DW9(phy)); + val &= ~IREF0RC_OFFSET_MASK; + val |= 0xE4 << IREF0RC_OFFSET_SHIFT; + I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val); + + val = I915_READ(BXT_PORT_CL1CM_DW10(phy)); + val &= ~IREF1RC_OFFSET_MASK; + val |= 0xE4 << IREF1RC_OFFSET_SHIFT; + I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val); + + /* Program power gating */ + val = I915_READ(BXT_PORT_CL1CM_DW28(phy)); + val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | + SUS_CLK_CONFIG; + I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val); + + if (phy_info->dual_channel) { + val = I915_READ(BXT_PORT_CL2CM_DW6(phy)); + val |= DW6_OLDO_DYN_PWR_DOWN_EN; + I915_WRITE(BXT_PORT_CL2CM_DW6(phy), val); + } + + if (phy_info->rcomp_phy != -1) { + uint32_t grc_code; + /* + * PHY0 isn't connected to an RCOMP resistor so copy over + * the corresponding calibrated value from PHY1, and disable + * the automatic calibration on PHY0. + */ + val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, + phy_info->rcomp_phy); + grc_code = val << GRC_CODE_FAST_SHIFT | + val << GRC_CODE_SLOW_SHIFT | + val; + I915_WRITE(BXT_PORT_REF_DW6(phy), grc_code); + + val = I915_READ(BXT_PORT_REF_DW8(phy)); + val |= GRC_DIS | GRC_RDY_OVRD; + I915_WRITE(BXT_PORT_REF_DW8(phy), val); + } + + val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); + val |= COMMON_RESET_DIS; + I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); + + if (phy_info->rcomp_phy == -1) + bxt_phy_wait_grc_done(dev_priv, phy); + +} + +void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) +{ + uint32_t val; + + val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); + val &= ~COMMON_RESET_DIS; + I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); + + val = I915_READ(BXT_P_CR_GT_DISP_PWRON); + val &= ~GT_DISPLAY_POWER_ON(phy); + I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); +} + +void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + enum dpio_phy rcomp_phy = phy_info->rcomp_phy; + bool was_enabled; + + lockdep_assert_held(&dev_priv->power_domains.lock); + + if (rcomp_phy != -1) { + was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy); + + /* + * We need to copy the GRC calibration value from rcomp_phy, + * so make sure it's powered up. + */ + if (!was_enabled) + _bxt_ddi_phy_init(dev_priv, rcomp_phy); + } + + _bxt_ddi_phy_init(dev_priv, phy); + + if (rcomp_phy != -1 && !was_enabled) + bxt_ddi_phy_uninit(dev_priv, phy_info->rcomp_phy); +} + +static bool __printf(6, 7) +__phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, + i915_reg_t reg, u32 mask, u32 expected, + const char *reg_fmt, ...) +{ + struct va_format vaf; + va_list args; + u32 val; + + val = I915_READ(reg); + if ((val & mask) == expected) + return true; + + va_start(args, reg_fmt); + vaf.fmt = reg_fmt; + vaf.va = &args; + + DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: " + "current %08x, expected %08x (mask %08x)\n", + phy, &vaf, reg.reg, val, (val & ~mask) | expected, + mask); + + va_end(args); + + return false; +} + +bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + uint32_t mask; + bool ok; + +#define _CHK(reg, mask, exp, fmt, ...) \ + __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt, \ + ## __VA_ARGS__) + + if (!bxt_ddi_phy_is_enabled(dev_priv, phy)) + return false; + + ok = true; + + /* PLL Rcomp code offset */ + ok &= _CHK(BXT_PORT_CL1CM_DW9(phy), + IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT, + "BXT_PORT_CL1CM_DW9(%d)", phy); + ok &= _CHK(BXT_PORT_CL1CM_DW10(phy), + IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT, + "BXT_PORT_CL1CM_DW10(%d)", phy); + + /* Power gating */ + mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG; + ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask, + "BXT_PORT_CL1CM_DW28(%d)", phy); + + if (phy_info->dual_channel) + ok &= _CHK(BXT_PORT_CL2CM_DW6(phy), + DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN, + "BXT_PORT_CL2CM_DW6(%d)", phy); + + if (phy_info->rcomp_phy != -1) { + u32 grc_code = dev_priv->bxt_phy_grc; + + grc_code = grc_code << GRC_CODE_FAST_SHIFT | + grc_code << GRC_CODE_SLOW_SHIFT | + grc_code; + mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK | + GRC_CODE_NOM_MASK; + ok &= _CHK(BXT_PORT_REF_DW6(phy), mask, grc_code, + "BXT_PORT_REF_DW6(%d)", phy); + + mask = GRC_DIS | GRC_RDY_OVRD; + ok &= _CHK(BXT_PORT_REF_DW8(phy), mask, mask, + "BXT_PORT_REF_DW8(%d)", phy); + } + + return ok; +#undef _CHK +} + +uint8_t +bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, + uint8_t lane_count) +{ + switch (lane_count) { + case 1: + return 0; + case 2: + return BIT(2) | BIT(0); + case 4: + return BIT(3) | BIT(2) | BIT(0); + default: + MISSING_CASE(lane_count); + + return 0; + } +} + +void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, + uint8_t lane_lat_optim_mask) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); + enum port port = dport->port; + enum dpio_phy phy; + enum dpio_channel ch; + int lane; + + bxt_port_to_phy_channel(port, &phy, &ch); + + for (lane = 0; lane < 4; lane++) { + u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); + + /* + * Note that on CHV this flag is called UPAR, but has + * the same function. + */ + val &= ~LATENCY_OPTIM; + if (lane_lat_optim_mask & BIT(lane)) + val |= LATENCY_OPTIM; + + I915_WRITE(BXT_PORT_TX_DW14_LN(phy, ch, lane), val); + } +} + +uint8_t +bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); + enum port port = dport->port; + enum dpio_phy phy; + enum dpio_channel ch; + int lane; + uint8_t mask; + + bxt_port_to_phy_channel(port, &phy, &ch); + + mask = 0; + for (lane = 0; lane < 4; lane++) { + u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); + + if (val & LATENCY_OPTIM) + mask |= BIT(lane); + } + + return mask; +} + + void chv_set_phy_signal_level(struct intel_encoder *encoder, u32 deemph_reg_value, u32 margin_reg_value, bool uniq_trans_scale) diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c index 605d0b509f24..58a756f2f224 100644 --- a/drivers/gpu/drm/i915/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c @@ -188,13 +188,12 @@ out: void intel_disable_shared_dpll(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_shared_dpll *pll = crtc->config->shared_dpll; unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base); /* PCH only available on ILK+ */ - if (INTEL_INFO(dev)->gen < 5) + if (INTEL_GEN(dev_priv) < 5) return; if (pll == NULL) @@ -1371,6 +1370,10 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, { uint32_t temp; enum port port = (enum port)pll->id; /* 1:1 port->PLL mapping */ + enum dpio_phy phy; + enum dpio_channel ch; + + bxt_port_to_phy_channel(port, &phy, &ch); /* Non-SSC reference */ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); @@ -1378,72 +1381,72 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); /* Disable 10 bit clock */ - temp = I915_READ(BXT_PORT_PLL_EBB_4(port)); + temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); temp &= ~PORT_PLL_10BIT_CLK_ENABLE; - I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); /* Write P1 & P2 */ - temp = I915_READ(BXT_PORT_PLL_EBB_0(port)); + temp = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch)); temp &= ~(PORT_PLL_P1_MASK | PORT_PLL_P2_MASK); temp |= pll->config.hw_state.ebb0; - I915_WRITE(BXT_PORT_PLL_EBB_0(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_0(phy, ch), temp); /* Write M2 integer */ - temp = I915_READ(BXT_PORT_PLL(port, 0)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 0)); temp &= ~PORT_PLL_M2_MASK; temp |= pll->config.hw_state.pll0; - I915_WRITE(BXT_PORT_PLL(port, 0), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 0), temp); /* Write N */ - temp = I915_READ(BXT_PORT_PLL(port, 1)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 1)); temp &= ~PORT_PLL_N_MASK; temp |= pll->config.hw_state.pll1; - I915_WRITE(BXT_PORT_PLL(port, 1), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 1), temp); /* Write M2 fraction */ - temp = I915_READ(BXT_PORT_PLL(port, 2)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 2)); temp &= ~PORT_PLL_M2_FRAC_MASK; temp |= pll->config.hw_state.pll2; - I915_WRITE(BXT_PORT_PLL(port, 2), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 2), temp); /* Write M2 fraction enable */ - temp = I915_READ(BXT_PORT_PLL(port, 3)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 3)); temp &= ~PORT_PLL_M2_FRAC_ENABLE; temp |= pll->config.hw_state.pll3; - I915_WRITE(BXT_PORT_PLL(port, 3), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 3), temp); /* Write coeff */ - temp = I915_READ(BXT_PORT_PLL(port, 6)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 6)); temp &= ~PORT_PLL_PROP_COEFF_MASK; temp &= ~PORT_PLL_INT_COEFF_MASK; temp &= ~PORT_PLL_GAIN_CTL_MASK; temp |= pll->config.hw_state.pll6; - I915_WRITE(BXT_PORT_PLL(port, 6), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 6), temp); /* Write calibration val */ - temp = I915_READ(BXT_PORT_PLL(port, 8)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 8)); temp &= ~PORT_PLL_TARGET_CNT_MASK; temp |= pll->config.hw_state.pll8; - I915_WRITE(BXT_PORT_PLL(port, 8), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 8), temp); - temp = I915_READ(BXT_PORT_PLL(port, 9)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 9)); temp &= ~PORT_PLL_LOCK_THRESHOLD_MASK; temp |= pll->config.hw_state.pll9; - I915_WRITE(BXT_PORT_PLL(port, 9), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 9), temp); - temp = I915_READ(BXT_PORT_PLL(port, 10)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 10)); temp &= ~PORT_PLL_DCO_AMP_OVR_EN_H; temp &= ~PORT_PLL_DCO_AMP_MASK; temp |= pll->config.hw_state.pll10; - I915_WRITE(BXT_PORT_PLL(port, 10), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 10), temp); /* Recalibrate with new settings */ - temp = I915_READ(BXT_PORT_PLL_EBB_4(port)); + temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); temp |= PORT_PLL_RECALIBRATE; - I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); temp &= ~PORT_PLL_10BIT_CLK_ENABLE; temp |= pll->config.hw_state.ebb4; - I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); /* Enable PLL */ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); @@ -1459,11 +1462,11 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, * While we write to the group register to program all lanes at once we * can read only lane registers and we pick lanes 0/1 for that. */ - temp = I915_READ(BXT_PORT_PCS_DW12_LN01(port)); + temp = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch)); temp &= ~LANE_STAGGER_MASK; temp &= ~LANESTAGGER_STRAP_OVRD; temp |= pll->config.hw_state.pcsdw12; - I915_WRITE(BXT_PORT_PCS_DW12_GRP(port), temp); + I915_WRITE(BXT_PORT_PCS_DW12_GRP(phy, ch), temp); } static void bxt_ddi_pll_disable(struct drm_i915_private *dev_priv, @@ -1485,6 +1488,10 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, enum port port = (enum port)pll->id; /* 1:1 port->PLL mapping */ uint32_t val; bool ret; + enum dpio_phy phy; + enum dpio_channel ch; + + bxt_port_to_phy_channel(port, &phy, &ch); if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) return false; @@ -1495,36 +1502,36 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, if (!(val & PORT_PLL_ENABLE)) goto out; - hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(port)); + hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch)); hw_state->ebb0 &= PORT_PLL_P1_MASK | PORT_PLL_P2_MASK; - hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(port)); + hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); hw_state->ebb4 &= PORT_PLL_10BIT_CLK_ENABLE; - hw_state->pll0 = I915_READ(BXT_PORT_PLL(port, 0)); + hw_state->pll0 = I915_READ(BXT_PORT_PLL(phy, ch, 0)); hw_state->pll0 &= PORT_PLL_M2_MASK; - hw_state->pll1 = I915_READ(BXT_PORT_PLL(port, 1)); + hw_state->pll1 = I915_READ(BXT_PORT_PLL(phy, ch, 1)); hw_state->pll1 &= PORT_PLL_N_MASK; - hw_state->pll2 = I915_READ(BXT_PORT_PLL(port, 2)); + hw_state->pll2 = I915_READ(BXT_PORT_PLL(phy, ch, 2)); hw_state->pll2 &= PORT_PLL_M2_FRAC_MASK; - hw_state->pll3 = I915_READ(BXT_PORT_PLL(port, 3)); + hw_state->pll3 = I915_READ(BXT_PORT_PLL(phy, ch, 3)); hw_state->pll3 &= PORT_PLL_M2_FRAC_ENABLE; - hw_state->pll6 = I915_READ(BXT_PORT_PLL(port, 6)); + hw_state->pll6 = I915_READ(BXT_PORT_PLL(phy, ch, 6)); hw_state->pll6 &= PORT_PLL_PROP_COEFF_MASK | PORT_PLL_INT_COEFF_MASK | PORT_PLL_GAIN_CTL_MASK; - hw_state->pll8 = I915_READ(BXT_PORT_PLL(port, 8)); + hw_state->pll8 = I915_READ(BXT_PORT_PLL(phy, ch, 8)); hw_state->pll8 &= PORT_PLL_TARGET_CNT_MASK; - hw_state->pll9 = I915_READ(BXT_PORT_PLL(port, 9)); + hw_state->pll9 = I915_READ(BXT_PORT_PLL(phy, ch, 9)); hw_state->pll9 &= PORT_PLL_LOCK_THRESHOLD_MASK; - hw_state->pll10 = I915_READ(BXT_PORT_PLL(port, 10)); + hw_state->pll10 = I915_READ(BXT_PORT_PLL(phy, ch, 10)); hw_state->pll10 &= PORT_PLL_DCO_AMP_OVR_EN_H | PORT_PLL_DCO_AMP_MASK; @@ -1533,11 +1540,11 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, * can read only lane registers. We configure all lanes the same way, so * here just read out lanes 0/1 and output a note if lanes 2/3 differ. */ - hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(port)); - if (I915_READ(BXT_PORT_PCS_DW12_LN23(port)) != hw_state->pcsdw12) + hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch)); + if (I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch)) != hw_state->pcsdw12) DRM_DEBUG_DRIVER("lane stagger config different for lane 01 (%08x) and 23 (%08x)\n", hw_state->pcsdw12, - I915_READ(BXT_PORT_PCS_DW12_LN23(port))); + I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch))); hw_state->pcsdw12 &= LANE_STAGGER_MASK | LANESTAGGER_STRAP_OVRD; ret = true; diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 4e90b0760f3f..cd132c216a67 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -294,6 +294,9 @@ struct intel_connector { */ struct intel_encoder *encoder; + /* ACPI device id for ACPI and driver cooperation */ + u32 acpi_device_id; + /* Reads out the current hw, returning true if the connector is enabled * and active (i.e. dpms ON state). */ bool (*get_hw_state)(struct intel_connector *); @@ -365,6 +368,8 @@ struct intel_atomic_state { /* Gen9+ only */ struct skl_wm_values wm_results; + + struct i915_sw_fence commit_ready; }; struct intel_plane_state { @@ -401,9 +406,6 @@ struct intel_plane_state { int scaler_id; struct drm_intel_sprite_colorkey ckey; - - /* async flip related structures */ - struct drm_i915_gem_request *wait_req; }; struct intel_initial_plane_config { @@ -501,14 +503,6 @@ struct intel_crtc_wm_state { /* gen9+ only needs 1-step wm programming */ struct skl_pipe_wm optimal; struct skl_ddb_entry ddb; - - /* cached plane data rate */ - unsigned plane_data_rate[I915_MAX_PLANES]; - unsigned plane_y_data_rate[I915_MAX_PLANES]; - - /* minimum block allocation */ - uint16_t minimum_blocks[I915_MAX_PLANES]; - uint16_t minimum_y_blocks[I915_MAX_PLANES]; } skl; }; @@ -661,7 +655,6 @@ struct intel_crtc_state { bool double_wide; - bool dp_encoder_is_mst; int pbn; struct intel_crtc_scaler_state scaler_state; @@ -731,16 +724,12 @@ struct intel_crtc { /* watermarks currently being used */ union { struct intel_pipe_wm ilk; - struct skl_pipe_wm skl; } active; /* allow CxSR on this pipe */ bool cxsr_allowed; } wm; - /* gen9+: ddb allocation currently being used */ - struct skl_ddb_entry hw_ddb; - int scanline_offset; struct { @@ -883,6 +872,14 @@ enum link_m_n_set { M2_N2 }; +struct intel_dp_desc { + u8 oui[3]; + u8 device_id[6]; + u8 hw_rev; + u8 sw_major_rev; + u8 sw_minor_rev; +} __packed; + struct intel_dp { i915_reg_t output_reg; i915_reg_t aux_ch_ctl_reg; @@ -905,6 +902,8 @@ struct intel_dp { /* sink rates as reported by DP_SUPPORTED_LINK_RATES */ uint8_t num_sink_rates; int sink_rates[DP_MAX_SUPPORTED_RATES]; + /* sink or branch descriptor */ + struct intel_dp_desc desc; struct drm_dp_aux aux; uint8_t train_set[4]; int panel_power_up_delay; @@ -964,7 +963,7 @@ struct intel_dp { struct intel_lspcon { bool active; enum drm_lspcon_mode mode; - struct drm_dp_aux *aux; + bool desc_valid; }; struct intel_digital_port { @@ -1028,17 +1027,15 @@ vlv_pipe_to_channel(enum pipe pipe) } } -static inline struct drm_crtc * -intel_get_crtc_for_pipe(struct drm_device *dev, int pipe) +static inline struct intel_crtc * +intel_get_crtc_for_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->pipe_to_crtc_mapping[pipe]; } -static inline struct drm_crtc * -intel_get_crtc_for_plane(struct drm_device *dev, int plane) +static inline struct intel_crtc * +intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum plane plane) { - struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->plane_to_crtc_mapping[plane]; } @@ -1098,15 +1095,6 @@ hdmi_to_dig_port(struct intel_hdmi *intel_hdmi) return container_of(intel_hdmi, struct intel_digital_port, hdmi); } -/* - * Returns the number of planes for this pipe, ie the number of sprites + 1 - * (primary plane). This doesn't count the cursor plane then. - */ -static inline unsigned int intel_num_planes(struct intel_crtc *crtc) -{ - return INTEL_INFO(crtc->base.dev)->num_sprites[crtc->pipe] + 1; -} - /* intel_fifo_underrun.c */ bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable); @@ -1123,6 +1111,9 @@ void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv); /* i915_irq.c */ void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); +void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 mask); +void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); +void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv); @@ -1145,6 +1136,9 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, unsigned int pipe_mask); void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, unsigned int pipe_mask); +void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv); +void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv); +void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv); /* intel_crt.c */ void intel_crt_init(struct drm_device *dev); @@ -1192,7 +1186,9 @@ u32 intel_fb_stride_alignment(const struct drm_i915_private *dev_priv, /* intel_audio.c */ void intel_init_audio_hooks(struct drm_i915_private *dev_priv); -void intel_audio_codec_enable(struct intel_encoder *encoder); +void intel_audio_codec_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); void intel_audio_codec_disable(struct intel_encoder *encoder); void i915_audio_component_init(struct drm_i915_private *dev_priv); void i915_audio_component_cleanup(struct drm_i915_private *dev_priv); @@ -1247,18 +1243,17 @@ intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state) (1 << INTEL_OUTPUT_EDP)); } static inline void -intel_wait_for_vblank(struct drm_device *dev, int pipe) +intel_wait_for_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) { - drm_wait_one_vblank(dev, pipe); + drm_wait_one_vblank(&dev_priv->drm, pipe); } static inline void -intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe) +intel_wait_for_vblank_if_active(struct drm_i915_private *dev_priv, int pipe) { - const struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + const struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->active) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); } u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc); @@ -1305,9 +1300,9 @@ unsigned int intel_tile_height(const struct drm_i915_private *dev_priv, void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv, enum pipe pipe); -int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, +int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, const struct dpll *dpll); -void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe); +void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe); int lpt_get_iclkip(struct drm_i915_private *dev_priv); /* modesetting asserts */ @@ -1335,12 +1330,6 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv); void hsw_disable_pc8(struct drm_i915_private *dev_priv); void bxt_init_cdclk(struct drm_i915_private *dev_priv); void bxt_uninit_cdclk(struct drm_i915_private *dev_priv); -void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); -void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); -bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, - enum dpio_phy phy); -bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, - enum dpio_phy phy); void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv); void bxt_enable_dc9(struct drm_i915_private *dev_priv); void bxt_disable_dc9(struct drm_i915_private *dev_priv); @@ -1358,7 +1347,7 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, struct dpll *best_clock); int chv_calc_dpll_params(int refclk, struct dpll *pll_clock); -bool intel_crtc_active(struct drm_crtc *crtc); +bool intel_crtc_active(struct intel_crtc *crtc); void hsw_enable_ips(struct intel_crtc *crtc); void hsw_disable_ips(struct intel_crtc *crtc); enum intel_display_power_domain @@ -1404,7 +1393,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc); bool intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state); -bool intel_dp_is_edp(struct drm_device *dev, enum port port); +bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port); enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd); void intel_edp_backlight_on(struct intel_dp *intel_dp); @@ -1451,6 +1440,11 @@ static inline unsigned int intel_dp_unused_lane_mask(int lane_count) return ~((1 << lane_count) - 1) & 0xf; } +bool intel_dp_read_dpcd(struct intel_dp *intel_dp); +bool __intel_dp_read_desc(struct intel_dp *intel_dp, + struct intel_dp_desc *desc); +bool intel_dp_read_desc(struct intel_dp *intel_dp); + /* intel_dp_aux_backlight.c */ int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector); @@ -1655,23 +1649,6 @@ assert_rpm_wakelock_held(struct drm_i915_private *dev_priv) DRM_DEBUG_DRIVER("RPM wakelock ref not held during HW access"); } -static inline int -assert_rpm_atomic_begin(struct drm_i915_private *dev_priv) -{ - int seq = atomic_read(&dev_priv->pm.atomic_seq); - - assert_rpm_wakelock_held(dev_priv); - - return seq; -} - -static inline void -assert_rpm_atomic_end(struct drm_i915_private *dev_priv, int begin_seq) -{ - WARN_ONCE(atomic_read(&dev_priv->pm.atomic_seq) != begin_seq, - "HW access outside of RPM atomic section\n"); -} - /** * disable_rpm_wakeref_asserts - disable the RPM assert checks * @dev_priv: i915 device instance @@ -1727,11 +1704,11 @@ bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, /* intel_pm.c */ -void intel_init_clock_gating(struct drm_device *dev); -void intel_suspend_hw(struct drm_device *dev); +void intel_init_clock_gating(struct drm_i915_private *dev_priv); +void intel_suspend_hw(struct drm_i915_private *dev_priv); int ilk_wm_max_level(const struct drm_i915_private *dev_priv); -void intel_update_watermarks(struct drm_crtc *crtc); -void intel_init_pm(struct drm_device *dev); +void intel_update_watermarks(struct intel_crtc *crtc); +void intel_init_pm(struct drm_i915_private *dev_priv); void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv); void intel_pm_setup(struct drm_device *dev); void intel_gpu_ips_init(struct drm_i915_private *dev_priv); @@ -1762,18 +1739,9 @@ int intel_enable_sagv(struct drm_i915_private *dev_priv); int intel_disable_sagv(struct drm_i915_private *dev_priv); bool skl_wm_level_equals(const struct skl_wm_level *l1, const struct skl_wm_level *l2); -bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old, - const struct skl_ddb_allocation *new, - enum pipe pipe); -bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state, - struct intel_crtc *intel_crtc); -void skl_write_cursor_wm(struct intel_crtc *intel_crtc, - const struct skl_plane_wm *wm, - const struct skl_ddb_allocation *ddb); -void skl_write_plane_wm(struct intel_crtc *intel_crtc, - const struct skl_plane_wm *wm, - const struct skl_ddb_allocation *ddb, - int plane); +bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries, + const struct skl_ddb_entry *ddb, + int ignore); uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config); bool ilk_disable_lp_wm(struct drm_device *dev); int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6); @@ -1790,7 +1758,8 @@ bool intel_sdvo_init(struct drm_device *dev, /* intel_sprite.c */ int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, int usecs); -int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane); +struct intel_plane *intel_sprite_plane_create(struct drm_i915_private *dev_priv, + enum pipe pipe, int plane); int intel_sprite_set_colorkey(struct drm_device *dev, void *data, struct drm_file *file_priv); void intel_pipe_update_start(struct intel_crtc *crtc); diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c index 4e0d025490a3..5b72c50d6f76 100644 --- a/drivers/gpu/drm/i915/intel_dsi.c +++ b/drivers/gpu/drm/i915/intel_dsi.c @@ -1298,12 +1298,6 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder, } } -static enum drm_connector_status -intel_dsi_detect(struct drm_connector *connector, bool force) -{ - return connector_status_connected; -} - static int intel_dsi_get_modes(struct drm_connector *connector) { struct intel_connector *intel_connector = to_intel_connector(connector); @@ -1407,7 +1401,6 @@ static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs static const struct drm_connector_funcs intel_dsi_connector_funcs = { .dpms = drm_atomic_helper_connector_dpms, - .detect = intel_dsi_detect, .late_register = intel_connector_register, .early_unregister = intel_connector_unregister, .destroy = intel_dsi_connector_destroy, diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c index 9f279a3d0f74..0d8ff0034b88 100644 --- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c +++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c @@ -774,9 +774,8 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id) 8); intel_dsi->clk_hs_to_lp_count += extra_byte_count; - DRM_DEBUG_KMS("Eot %s\n", intel_dsi->eotp_pkt ? "enabled" : "disabled"); - DRM_DEBUG_KMS("Clockstop %s\n", intel_dsi->clock_stop ? - "disabled" : "enabled"); + DRM_DEBUG_KMS("Eot %s\n", enableddisabled(intel_dsi->eotp_pkt)); + DRM_DEBUG_KMS("Clockstop %s\n", enableddisabled(!intel_dsi->clock_stop)); DRM_DEBUG_KMS("Mode %s\n", intel_dsi->operation_mode ? "command" : "video"); if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) DRM_DEBUG_KMS("Dual link: DSI_DUAL_LINK_FRONT_BACK\n"); @@ -795,8 +794,7 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id) DRM_DEBUG_KMS("LP to HS Clock Count 0x%x\n", intel_dsi->clk_lp_to_hs_count); DRM_DEBUG_KMS("HS to LP Clock Count 0x%x\n", intel_dsi->clk_hs_to_lp_count); DRM_DEBUG_KMS("BTA %s\n", - intel_dsi->video_frmt_cfg_bits & DISABLE_VIDEO_BTA ? - "disabled" : "enabled"); + enableddisabled(!(intel_dsi->video_frmt_cfg_bits & DISABLE_VIDEO_BTA))); /* delays in VBT are in unit of 100us, so need to convert * here in ms diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index cd574900cd8d..708645443046 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -393,12 +393,12 @@ intel_dvo_get_current_mode(struct drm_connector *connector) * its timings to get how the BIOS set up the panel. */ if (dvo_val & DVO_ENABLE) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; int pipe = (dvo_val & DVO_PIPE_B_SELECT) ? 1 : 0; - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc) { - mode = intel_crtc_mode_get(dev, crtc); + mode = intel_crtc_mode_get(dev, &crtc->base); if (mode) { mode->type |= DRM_MODE_TYPE_PREFERRED; if (dvo_val & DVO_HSYNC_ACTIVE_HIGH) diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c index 8cceb345aa0f..3da4d466e332 100644 --- a/drivers/gpu/drm/i915/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/intel_engine_cs.c @@ -102,6 +102,9 @@ intel_engine_setup(struct drm_i915_private *dev_priv, engine->mmio_base = info->mmio_base; engine->irq_shift = info->irq_shift; + /* Nothing to do here, execute in order of dependencies */ + engine->schedule = NULL; + dev_priv->engine[id] = engine; return 0; } @@ -174,7 +177,7 @@ cleanup: return ret; } -void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) +void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno) { struct drm_i915_private *dev_priv = engine->i915; @@ -204,13 +207,13 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) I915_NUM_ENGINES * gen8_semaphore_seqno_size); kunmap(page); } - memset(engine->semaphore.sync_seqno, 0, - sizeof(engine->semaphore.sync_seqno)); intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); if (engine->irq_seqno_barrier) engine->irq_seqno_barrier(engine); - engine->last_submitted_seqno = seqno; + + GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request)); + engine->timeline->last_submitted_seqno = seqno; engine->hangcheck.seqno = seqno; @@ -220,15 +223,9 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) intel_engine_wakeup(engine); } -void intel_engine_init_hangcheck(struct intel_engine_cs *engine) -{ - memset(&engine->hangcheck, 0, sizeof(engine->hangcheck)); -} - -static void intel_engine_init_requests(struct intel_engine_cs *engine) +static void intel_engine_init_timeline(struct intel_engine_cs *engine) { - init_request_active(&engine->last_request, NULL); - INIT_LIST_HEAD(&engine->request_list); + engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id]; } /** @@ -242,12 +239,10 @@ static void intel_engine_init_requests(struct intel_engine_cs *engine) */ void intel_engine_setup_common(struct intel_engine_cs *engine) { - INIT_LIST_HEAD(&engine->execlist_queue); - spin_lock_init(&engine->execlist_lock); + engine->execlist_queue = RB_ROOT; + engine->execlist_first = NULL; - engine->fence_context = dma_fence_context_alloc(1); - - intel_engine_init_requests(engine); + intel_engine_init_timeline(engine); intel_engine_init_hangcheck(engine); i915_gem_batch_pool_init(engine, &engine->batch_pool); @@ -264,7 +259,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size) obj = i915_gem_object_create_stolen(&engine->i915->drm, size); if (!obj) - obj = i915_gem_object_create(&engine->i915->drm, size); + obj = i915_gem_object_create_internal(engine->i915, size); if (IS_ERR(obj)) { DRM_ERROR("Failed to allocate scratch page\n"); return PTR_ERR(obj); @@ -314,6 +309,10 @@ int intel_engine_init_common(struct intel_engine_cs *engine) if (ret) return ret; + ret = i915_gem_render_state_init(engine); + if (ret) + return ret; + return 0; } @@ -328,6 +327,7 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) { intel_engine_cleanup_scratch(engine); + i915_gem_render_state_fini(engine); intel_engine_fini_breadcrumbs(engine); intel_engine_cleanup_cmd_parser(engine); i915_gem_batch_pool_fini(&engine->batch_pool); diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c index cbe2ebda4c40..62f215b12eb5 100644 --- a/drivers/gpu/drm/i915/intel_fbc.c +++ b/drivers/gpu/drm/i915/intel_fbc.c @@ -48,17 +48,17 @@ static inline bool fbc_supported(struct drm_i915_private *dev_priv) static inline bool fbc_on_pipe_a_only(struct drm_i915_private *dev_priv) { - return IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8; + return IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8; } static inline bool fbc_on_plane_a_only(struct drm_i915_private *dev_priv) { - return INTEL_INFO(dev_priv)->gen < 4; + return INTEL_GEN(dev_priv) < 4; } static inline bool no_fbc_on_multiple_pipes(struct drm_i915_private *dev_priv) { - return INTEL_INFO(dev_priv)->gen <= 3; + return INTEL_GEN(dev_priv) <= 3; } /* @@ -351,7 +351,7 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv) static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv) { - if (INTEL_INFO(dev_priv)->gen >= 5) + if (INTEL_GEN(dev_priv) >= 5) return ilk_fbc_is_active(dev_priv); else if (IS_GM45(dev_priv)) return g4x_fbc_is_active(dev_priv); @@ -365,9 +365,9 @@ static void intel_fbc_hw_activate(struct drm_i915_private *dev_priv) fbc->active = true; - if (INTEL_INFO(dev_priv)->gen >= 7) + if (INTEL_GEN(dev_priv) >= 7) gen7_fbc_activate(dev_priv); - else if (INTEL_INFO(dev_priv)->gen >= 5) + else if (INTEL_GEN(dev_priv) >= 5) ilk_fbc_activate(dev_priv); else if (IS_GM45(dev_priv)) g4x_fbc_activate(dev_priv); @@ -381,7 +381,7 @@ static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv) fbc->active = false; - if (INTEL_INFO(dev_priv)->gen >= 5) + if (INTEL_GEN(dev_priv) >= 5) ilk_fbc_deactivate(dev_priv); else if (IS_GM45(dev_priv)) g4x_fbc_deactivate(dev_priv); @@ -561,7 +561,7 @@ again: ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1, 4096, 0, end); - if (ret && INTEL_INFO(dev_priv)->gen <= 4) { + if (ret && INTEL_GEN(dev_priv) <= 4) { return 0; } else if (ret) { compression_threshold <<= 1; @@ -594,7 +594,7 @@ static int intel_fbc_alloc_cfb(struct intel_crtc *crtc) fbc->threshold = ret; - if (INTEL_INFO(dev_priv)->gen >= 5) + if (INTEL_GEN(dev_priv) >= 5) I915_WRITE(ILK_DPFC_CB_BASE, fbc->compressed_fb.start); else if (IS_GM45(dev_priv)) { I915_WRITE(DPFC_CB_BASE, fbc->compressed_fb.start); @@ -708,10 +708,10 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc) struct intel_fbc *fbc = &dev_priv->fbc; unsigned int effective_w, effective_h, max_w, max_h; - if (INTEL_INFO(dev_priv)->gen >= 8 || IS_HASWELL(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) { max_w = 4096; max_h = 4096; - } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) { + } else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { max_w = 4096; max_h = 2048; } else { @@ -812,7 +812,7 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) fbc->no_fbc_reason = "framebuffer not tiled or fenced"; return false; } - if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_G4X(dev_priv) && + if (INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv) && cache->plane.rotation != DRM_ROTATE_0) { fbc->no_fbc_reason = "rotation unsupported"; return false; @@ -854,9 +854,8 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) return true; } -static bool intel_fbc_can_choose(struct intel_crtc *crtc) +static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; if (intel_vgpu_active(dev_priv)) { @@ -874,16 +873,6 @@ static bool intel_fbc_can_choose(struct intel_crtc *crtc) return false; } - if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A) { - fbc->no_fbc_reason = "no enabled pipes can have FBC"; - return false; - } - - if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A) { - fbc->no_fbc_reason = "no enabled planes can have FBC"; - return false; - } - return true; } @@ -1066,23 +1055,19 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv, struct drm_atomic_state *state) { struct intel_fbc *fbc = &dev_priv->fbc; - struct drm_crtc *crtc; - struct drm_crtc_state *crtc_state; struct drm_plane *plane; struct drm_plane_state *plane_state; - bool fbc_crtc_present = false; - int i, j; + bool crtc_chosen = false; + int i; mutex_lock(&fbc->lock); - for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (fbc->crtc == to_intel_crtc(crtc)) { - fbc_crtc_present = true; - break; - } - } - /* This atomic commit doesn't involve the CRTC currently tied to FBC. */ - if (!fbc_crtc_present && fbc->crtc != NULL) + /* Does this atomic commit involve the CRTC currently tied to FBC? */ + if (fbc->crtc && + !drm_atomic_get_existing_crtc_state(state, &fbc->crtc->base)) + goto out; + + if (!intel_fbc_can_enable(dev_priv)) goto out; /* Simply choose the first CRTC that is compatible and has a visible @@ -1092,25 +1077,29 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv, for_each_plane_in_state(state, plane, plane_state, i) { struct intel_plane_state *intel_plane_state = to_intel_plane_state(plane_state); + struct intel_crtc_state *intel_crtc_state; + struct intel_crtc *crtc = to_intel_crtc(plane_state->crtc); if (!intel_plane_state->base.visible) continue; - for_each_crtc_in_state(state, crtc, crtc_state, j) { - struct intel_crtc_state *intel_crtc_state = - to_intel_crtc_state(crtc_state); + if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A) + continue; - if (plane_state->crtc != crtc) - continue; + if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A) + continue; - if (!intel_fbc_can_choose(to_intel_crtc(crtc))) - break; + intel_crtc_state = to_intel_crtc_state( + drm_atomic_get_existing_crtc_state(state, &crtc->base)); - intel_crtc_state->enable_fbc = true; - goto out; - } + intel_crtc_state->enable_fbc = true; + crtc_chosen = true; + break; } + if (!crtc_chosen) + fbc->no_fbc_reason = "no suitable CRTC for FBC"; + out: mutex_unlock(&fbc->lock); } @@ -1306,7 +1295,7 @@ void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv) return; for_each_intel_crtc(&dev_priv->drm, crtc) - if (intel_crtc_active(&crtc->base) && + if (intel_crtc_active(crtc) && to_intel_plane_state(crtc->base.primary->state)->base.visible) dev_priv->fbc.visible_pipes_mask |= (1 << crtc->pipe); } @@ -1386,7 +1375,7 @@ void intel_fbc_init(struct drm_i915_private *dev_priv) } /* This value was pulled out of someone's hat */ - if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_GM45(dev_priv)) + if (INTEL_GEN(dev_priv) <= 4 && !IS_GM45(dev_priv)) I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); /* We still don't have any sort of hardware state readout for FBC, so diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index b7098f98bb67..beb08982dc0b 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -102,16 +102,13 @@ static int intel_fbdev_pan_display(struct fb_var_screeninfo *var, static struct fb_ops intelfb_ops = { .owner = THIS_MODULE, - .fb_check_var = drm_fb_helper_check_var, + DRM_FB_HELPER_DEFAULT_OPS, .fb_set_par = intel_fbdev_set_par, .fb_fillrect = drm_fb_helper_cfb_fillrect, .fb_copyarea = drm_fb_helper_cfb_copyarea, .fb_imageblit = drm_fb_helper_cfb_imageblit, .fb_pan_display = intel_fbdev_pan_display, .fb_blank = intel_fbdev_blank, - .fb_setcmap = drm_fb_helper_setcmap, - .fb_debug_enter = drm_fb_helper_debug_enter, - .fb_debug_leave = drm_fb_helper_debug_leave, }; static int intelfb_alloc(struct drm_fb_helper *helper, @@ -359,7 +356,7 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, struct drm_fb_offset *offsets, bool *enabled, int width, int height) { - struct drm_device *dev = fb_helper->dev; + struct drm_i915_private *dev_priv = to_i915(fb_helper->dev); unsigned long conn_configured, mask; unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG); int i, j; @@ -512,7 +509,7 @@ retry: * fbdev helper library. */ if (num_connectors_enabled != num_connectors_detected && - num_connectors_enabled < INTEL_INFO(dev)->num_pipes) { + num_connectors_enabled < INTEL_INFO(dev_priv)->num_pipes) { DRM_DEBUG_KMS("fallback: Not all outputs enabled\n"); DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled, num_connectors_detected); @@ -636,7 +633,7 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, cur_size = intel_crtc->config->base.adjusted_mode.crtc_vdisplay; cur_size = intel_fb_align_height(dev, cur_size, fb->base.pixel_format, - fb->base.modifier[0]); + fb->base.modifier); cur_size *= fb->base.pitches[0]; DRM_DEBUG_KMS("pipe %c area: %dx%d, bpp: %d, size: %d\n", pipe_name(intel_crtc->pipe), @@ -700,11 +697,11 @@ static void intel_fbdev_suspend_worker(struct work_struct *work) int intel_fbdev_init(struct drm_device *dev) { - struct intel_fbdev *ifbdev; struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_fbdev *ifbdev; int ret; - if (WARN_ON(INTEL_INFO(dev)->num_pipes == 0)) + if (WARN_ON(INTEL_INFO(dev_priv)->num_pipes == 0)) return -ENODEV; ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL); @@ -717,7 +714,7 @@ int intel_fbdev_init(struct drm_device *dev) ifbdev->preferred_bpp = 32; ret = drm_fb_helper_init(dev, &ifbdev->helper, - INTEL_INFO(dev)->num_pipes, 4); + INTEL_INFO(dev_priv)->num_pipes, 4); if (ret) { kfree(ifbdev); return ret; diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c index 3018f4f589c8..e660d8b4bbc3 100644 --- a/drivers/gpu/drm/i915/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c @@ -57,7 +57,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->cpu_fifo_underrun_disabled) return false; @@ -75,7 +75,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->pch_fifo_underrun_disabled) return false; @@ -245,14 +245,13 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); bool old; assert_spin_locked(&dev_priv->irq_lock); - old = !intel_crtc->cpu_fifo_underrun_disabled; - intel_crtc->cpu_fifo_underrun_disabled = !enable; + old = !crtc->cpu_fifo_underrun_disabled; + crtc->cpu_fifo_underrun_disabled = !enable; if (HAS_GMCH_DISPLAY(dev_priv)) i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old); @@ -314,8 +313,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, enum transcoder pch_transcoder, bool enable) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = + intel_get_crtc_for_pipe(dev_priv, (enum pipe) pch_transcoder); unsigned long flags; bool old; @@ -330,8 +329,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, spin_lock_irqsave(&dev_priv->irq_lock, flags); - old = !intel_crtc->pch_fifo_underrun_disabled; - intel_crtc->pch_fifo_underrun_disabled = !enable; + old = !crtc->pch_fifo_underrun_disabled; + crtc->pch_fifo_underrun_disabled = !enable; if (HAS_PCH_IBX(dev_priv)) ibx_set_fifo_underrun_reporting(&dev_priv->drm, @@ -358,7 +357,7 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); /* We may be called too early in init, thanks BIOS! */ if (crtc == NULL) @@ -366,7 +365,7 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, /* GMCH can't disable fifo underruns, filter them. */ if (HAS_GMCH_DISPLAY(dev_priv) && - to_intel_crtc(crtc)->cpu_fifo_underrun_disabled) + crtc->cpu_fifo_underrun_disabled) return; if (intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false)) diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.h b/drivers/gpu/drm/i915/intel_frontbuffer.h index 76ceb539f9f0..7bab41218cf7 100644 --- a/drivers/gpu/drm/i915/intel_frontbuffer.h +++ b/drivers/gpu/drm/i915/intel_frontbuffer.h @@ -53,16 +53,17 @@ void __intel_fb_obj_flush(struct drm_i915_gem_object *obj, * until the rendering completes or a flip on this frontbuffer plane is * scheduled. */ -static inline void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, +static inline bool intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, enum fb_op_origin origin) { unsigned int frontbuffer_bits; frontbuffer_bits = atomic_read(&obj->frontbuffer_bits); if (!frontbuffer_bits) - return; + return false; __intel_fb_obj_invalidate(obj, origin, frontbuffer_bits); + return true; } /** diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h index 5cdf7aa75be5..0053258e03d3 100644 --- a/drivers/gpu/drm/i915/intel_guc.h +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -64,7 +64,7 @@ struct drm_i915_gem_request; */ struct i915_guc_client { struct i915_vma *vma; - void *client_base; /* first page (only) of above */ + void *vaddr; struct i915_gem_context *owner; struct intel_guc *guc; @@ -123,10 +123,28 @@ struct intel_guc_fw { uint32_t ucode_offset; }; +struct intel_guc_log { + uint32_t flags; + struct i915_vma *vma; + void *buf_addr; + struct workqueue_struct *flush_wq; + struct work_struct flush_work; + struct rchan *relay_chan; + + /* logging related stats */ + u32 capture_miss_count; + u32 flush_interrupt_count; + u32 prev_overflow_count[GUC_MAX_LOG_BUFFER]; + u32 total_overflow_count[GUC_MAX_LOG_BUFFER]; + u32 flush_count[GUC_MAX_LOG_BUFFER]; +}; + struct intel_guc { struct intel_guc_fw guc_fw; - uint32_t log_flags; - struct i915_vma *log_vma; + struct intel_guc_log log; + + /* GuC2Host interrupt related state */ + bool interrupts_enabled; struct i915_vma *ads_vma; struct i915_vma *ctx_pool_vma; @@ -146,6 +164,9 @@ struct intel_guc { uint64_t submissions[I915_NUM_ENGINES]; uint32_t last_seqno[I915_NUM_ENGINES]; + + /* To serialize the Host2GuC actions */ + struct mutex action_lock; }; /* intel_guc_loader.c */ @@ -163,5 +184,10 @@ int i915_guc_wq_reserve(struct drm_i915_gem_request *rq); void i915_guc_wq_unreserve(struct drm_i915_gem_request *request); void i915_guc_submission_disable(struct drm_i915_private *dev_priv); void i915_guc_submission_fini(struct drm_i915_private *dev_priv); +void i915_guc_capture_logs(struct drm_i915_private *dev_priv); +void i915_guc_flush_logs(struct drm_i915_private *dev_priv); +void i915_guc_register(struct drm_i915_private *dev_priv); +void i915_guc_unregister(struct drm_i915_private *dev_priv); +int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val); #endif diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h index e40db2d2ae99..324ea902558b 100644 --- a/drivers/gpu/drm/i915/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/intel_guc_fwif.h @@ -104,9 +104,9 @@ #define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) #define GUC_LOG_CRASH_PAGES 1 #define GUC_LOG_CRASH_SHIFT 4 -#define GUC_LOG_DPC_PAGES 3 +#define GUC_LOG_DPC_PAGES 7 #define GUC_LOG_DPC_SHIFT 6 -#define GUC_LOG_ISR_PAGES 3 +#define GUC_LOG_ISR_PAGES 7 #define GUC_LOG_ISR_SHIFT 9 #define GUC_LOG_BUF_ADDR_SHIFT 12 @@ -419,15 +419,87 @@ struct guc_ads { u32 reserved2[4]; } __packed; +/* GuC logging structures */ + +enum guc_log_buffer_type { + GUC_ISR_LOG_BUFFER, + GUC_DPC_LOG_BUFFER, + GUC_CRASH_DUMP_LOG_BUFFER, + GUC_MAX_LOG_BUFFER +}; + +/** + * DOC: GuC Log buffer Layout + * + * Page0 +-------------------------------+ + * | ISR state header (32 bytes) | + * | DPC state header | + * | Crash dump state header | + * Page1 +-------------------------------+ + * | ISR logs | + * Page9 +-------------------------------+ + * | DPC logs | + * Page17 +-------------------------------+ + * | Crash Dump logs | + * +-------------------------------+ + * + * Below state structure is used for coordination of retrieval of GuC firmware + * logs. Separate state is maintained for each log buffer type. + * read_ptr points to the location where i915 read last in log buffer and + * is read only for GuC firmware. write_ptr is incremented by GuC with number + * of bytes written for each log entry and is read only for i915. + * When any type of log buffer becomes half full, GuC sends a flush interrupt. + * GuC firmware expects that while it is writing to 2nd half of the buffer, + * first half would get consumed by Host and then get a flush completed + * acknowledgment from Host, so that it does not end up doing any overwrite + * causing loss of logs. So when buffer gets half filled & i915 has requested + * for interrupt, GuC will set flush_to_file field, set the sampled_write_ptr + * to the value of write_ptr and raise the interrupt. + * On receiving the interrupt i915 should read the buffer, clear flush_to_file + * field and also update read_ptr with the value of sample_write_ptr, before + * sending an acknowledgment to GuC. marker & version fields are for internal + * usage of GuC and opaque to i915. buffer_full_cnt field is incremented every + * time GuC detects the log buffer overflow. + */ +struct guc_log_buffer_state { + u32 marker[2]; + u32 read_ptr; + u32 write_ptr; + u32 size; + u32 sampled_write_ptr; + union { + struct { + u32 flush_to_file:1; + u32 buffer_full_cnt:4; + u32 reserved:27; + }; + u32 flags; + }; + u32 version; +} __packed; + +union guc_log_control { + struct { + u32 logging_enabled:1; + u32 reserved1:3; + u32 verbosity:4; + u32 reserved2:24; + }; + u32 value; +} __packed; + /* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */ enum host2guc_action { HOST2GUC_ACTION_DEFAULT = 0x0, HOST2GUC_ACTION_SAMPLE_FORCEWAKE = 0x6, HOST2GUC_ACTION_ALLOCATE_DOORBELL = 0x10, HOST2GUC_ACTION_DEALLOCATE_DOORBELL = 0x20, + HOST2GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE = 0x30, + HOST2GUC_ACTION_FORCE_LOG_BUFFER_FLUSH = 0x302, HOST2GUC_ACTION_ENTER_S_STATE = 0x501, HOST2GUC_ACTION_EXIT_S_STATE = 0x502, HOST2GUC_ACTION_SLPC_REQUEST = 0x3003, + HOST2GUC_ACTION_UK_LOG_ENABLE_LOGGING = 0x0E000, HOST2GUC_ACTION_LIMIT }; @@ -449,4 +521,10 @@ enum guc2host_status { GUC2HOST_STATUS_GENERIC_FAIL = GUC2HOST_STATUS(0x0000F000) }; +/* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */ +enum guc2host_message { + GUC2HOST_MSG_CRASH_DUMP_POSTED = (1 << 1), + GUC2HOST_MSG_FLUSH_LOG_BUFFER = (1 << 3) +}; + #endif diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c index 3c8eaae13732..34d6ad2cf7c1 100644 --- a/drivers/gpu/drm/i915/intel_guc_loader.c +++ b/drivers/gpu/drm/i915/intel_guc_loader.c @@ -211,11 +211,13 @@ static void guc_params_init(struct drm_i915_private *dev_priv) params[GUC_CTL_FEATURE] |= GUC_CTL_DISABLE_SCHEDULER | GUC_CTL_VCS2_ENABLED; + params[GUC_CTL_LOG_PARAMS] = guc->log.flags; + if (i915.guc_log_level >= 0) { - params[GUC_CTL_LOG_PARAMS] = guc->log_flags; params[GUC_CTL_DEBUG] = i915.guc_log_level << GUC_LOG_VERBOSITY_SHIFT; - } + } else + params[GUC_CTL_DEBUG] = GUC_LOG_DISABLED; if (guc->ads_vma) { u32 ads = i915_ggtt_offset(guc->ads_vma) >> PAGE_SHIFT; @@ -483,6 +485,7 @@ int intel_guc_setup(struct drm_device *dev) } guc_interrupts_release(dev_priv); + gen9_reset_guc_interrupts(dev_priv); guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING; @@ -527,6 +530,9 @@ int intel_guc_setup(struct drm_device *dev) intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); if (i915.enable_guc_submission) { + if (i915.guc_log_level >= 0) + gen9_enable_guc_interrupts(dev_priv); + err = i915_guc_submission_enable(dev_priv); if (err) goto fail; @@ -560,7 +566,7 @@ fail: ret = 0; } - if (err == 0 && !HAS_GUC_UCODE(dev)) + if (err == 0 && !HAS_GUC_UCODE(dev_priv)) ; /* Don't mention the GuC! */ else if (err == 0) DRM_INFO("GuC firmware load skipped\n"); @@ -719,18 +725,18 @@ void intel_guc_init(struct drm_device *dev) struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; const char *fw_path; - if (!HAS_GUC(dev)) { + if (!HAS_GUC(dev_priv)) { i915.enable_guc_loading = 0; i915.enable_guc_submission = 0; } else { /* A negative value means "use platform default" */ if (i915.enable_guc_loading < 0) - i915.enable_guc_loading = HAS_GUC_UCODE(dev); + i915.enable_guc_loading = HAS_GUC_UCODE(dev_priv); if (i915.enable_guc_submission < 0) - i915.enable_guc_submission = HAS_GUC_SCHED(dev); + i915.enable_guc_submission = HAS_GUC_SCHED(dev_priv); } - if (!HAS_GUC_UCODE(dev)) { + if (!HAS_GUC_UCODE(dev_priv)) { fw_path = NULL; } else if (IS_SKYLAKE(dev_priv)) { fw_path = I915_SKL_GUC_UCODE; diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c new file mode 100644 index 000000000000..53df5b11bff4 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_hangcheck.c @@ -0,0 +1,450 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "i915_drv.h" + +static bool +ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr) +{ + if (INTEL_GEN(engine->i915) >= 8) { + return (ipehr >> 23) == 0x1c; + } else { + ipehr &= ~MI_SEMAPHORE_SYNC_MASK; + return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | + MI_SEMAPHORE_REGISTER); + } +} + +static struct intel_engine_cs * +semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr, + u64 offset) +{ + struct drm_i915_private *dev_priv = engine->i915; + struct intel_engine_cs *signaller; + enum intel_engine_id id; + + if (INTEL_GEN(dev_priv) >= 8) { + for_each_engine(signaller, dev_priv, id) { + if (engine == signaller) + continue; + + if (offset == signaller->semaphore.signal_ggtt[engine->hw_id]) + return signaller; + } + } else { + u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; + + for_each_engine(signaller, dev_priv, id) { + if(engine == signaller) + continue; + + if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id]) + return signaller; + } + } + + DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n", + engine->name, ipehr, offset); + + return ERR_PTR(-ENODEV); +} + +static struct intel_engine_cs * +semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) +{ + struct drm_i915_private *dev_priv = engine->i915; + void __iomem *vaddr; + u32 cmd, ipehr, head; + u64 offset = 0; + int i, backwards; + + /* + * This function does not support execlist mode - any attempt to + * proceed further into this function will result in a kernel panic + * when dereferencing ring->buffer, which is not set up in execlist + * mode. + * + * The correct way of doing it would be to derive the currently + * executing ring buffer from the current context, which is derived + * from the currently running request. Unfortunately, to get the + * current request we would have to grab the struct_mutex before doing + * anything else, which would be ill-advised since some other thread + * might have grabbed it already and managed to hang itself, causing + * the hang checker to deadlock. + * + * Therefore, this function does not support execlist mode in its + * current form. Just return NULL and move on. + */ + if (engine->buffer == NULL) + return NULL; + + ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); + if (!ipehr_is_semaphore_wait(engine, ipehr)) + return NULL; + + /* + * HEAD is likely pointing to the dword after the actual command, + * so scan backwards until we find the MBOX. But limit it to just 3 + * or 4 dwords depending on the semaphore wait command size. + * Note that we don't care about ACTHD here since that might + * point at at batch, and semaphores are always emitted into the + * ringbuffer itself. + */ + head = I915_READ_HEAD(engine) & HEAD_ADDR; + backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4; + vaddr = (void __iomem *)engine->buffer->vaddr; + + for (i = backwards; i; --i) { + /* + * Be paranoid and presume the hw has gone off into the wild - + * our ring is smaller than what the hardware (and hence + * HEAD_ADDR) allows. Also handles wrap-around. + */ + head &= engine->buffer->size - 1; + + /* This here seems to blow up */ + cmd = ioread32(vaddr + head); + if (cmd == ipehr) + break; + + head -= 4; + } + + if (!i) + return NULL; + + *seqno = ioread32(vaddr + head + 4) + 1; + if (INTEL_GEN(dev_priv) >= 8) { + offset = ioread32(vaddr + head + 12); + offset <<= 32; + offset |= ioread32(vaddr + head + 8); + } + return semaphore_wait_to_signaller_ring(engine, ipehr, offset); +} + +static int semaphore_passed(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + struct intel_engine_cs *signaller; + u32 seqno; + + engine->hangcheck.deadlock++; + + signaller = semaphore_waits_for(engine, &seqno); + if (signaller == NULL) + return -1; + + if (IS_ERR(signaller)) + return 0; + + /* Prevent pathological recursion due to driver bugs */ + if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES) + return -1; + + if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno)) + return 1; + + /* cursory check for an unkickable deadlock */ + if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE && + semaphore_passed(signaller) < 0) + return -1; + + return 0; +} + +static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, dev_priv, id) + engine->hangcheck.deadlock = 0; +} + +static bool instdone_unchanged(u32 current_instdone, u32 *old_instdone) +{ + u32 tmp = current_instdone | *old_instdone; + bool unchanged; + + unchanged = tmp == *old_instdone; + *old_instdone |= tmp; + + return unchanged; +} + +static bool subunits_stuck(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + struct intel_instdone instdone; + struct intel_instdone *accu_instdone = &engine->hangcheck.instdone; + bool stuck; + int slice; + int subslice; + + if (engine->id != RCS) + return true; + + intel_engine_get_instdone(engine, &instdone); + + /* There might be unstable subunit states even when + * actual head is not moving. Filter out the unstable ones by + * accumulating the undone -> done transitions and only + * consider those as progress. + */ + stuck = instdone_unchanged(instdone.instdone, + &accu_instdone->instdone); + stuck &= instdone_unchanged(instdone.slice_common, + &accu_instdone->slice_common); + + for_each_instdone_slice_subslice(dev_priv, slice, subslice) { + stuck &= instdone_unchanged(instdone.sampler[slice][subslice], + &accu_instdone->sampler[slice][subslice]); + stuck &= instdone_unchanged(instdone.row[slice][subslice], + &accu_instdone->row[slice][subslice]); + } + + return stuck; +} + +static enum intel_engine_hangcheck_action +head_stuck(struct intel_engine_cs *engine, u64 acthd) +{ + if (acthd != engine->hangcheck.acthd) { + + /* Clear subunit states on head movement */ + memset(&engine->hangcheck.instdone, 0, + sizeof(engine->hangcheck.instdone)); + + return HANGCHECK_ACTIVE; + } + + if (!subunits_stuck(engine)) + return HANGCHECK_ACTIVE; + + return HANGCHECK_HUNG; +} + +static enum intel_engine_hangcheck_action +engine_stuck(struct intel_engine_cs *engine, u64 acthd) +{ + struct drm_i915_private *dev_priv = engine->i915; + enum intel_engine_hangcheck_action ha; + u32 tmp; + + ha = head_stuck(engine, acthd); + if (ha != HANGCHECK_HUNG) + return ha; + + if (IS_GEN2(dev_priv)) + return HANGCHECK_HUNG; + + /* Is the chip hanging on a WAIT_FOR_EVENT? + * If so we can simply poke the RB_WAIT bit + * and break the hang. This should work on + * all but the second generation chipsets. + */ + tmp = I915_READ_CTL(engine); + if (tmp & RING_WAIT) { + i915_handle_error(dev_priv, 0, + "Kicking stuck wait on %s", + engine->name); + I915_WRITE_CTL(engine, tmp); + return HANGCHECK_KICK; + } + + if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) { + switch (semaphore_passed(engine)) { + default: + return HANGCHECK_HUNG; + case 1: + i915_handle_error(dev_priv, 0, + "Kicking stuck semaphore on %s", + engine->name); + I915_WRITE_CTL(engine, tmp); + return HANGCHECK_KICK; + case 0: + return HANGCHECK_WAIT; + } + } + + return HANGCHECK_HUNG; +} + +/* + * This is called when the chip hasn't reported back with completed + * batchbuffers in a long time. We keep track per ring seqno progress and + * if there are no progress, hangcheck score for that ring is increased. + * Further, acthd is inspected to see if the ring is stuck. On stuck case + * we kick the ring. If we see no progress on three subsequent calls + * we assume chip is wedged and try to fix it by resetting the chip. + */ +static void i915_hangcheck_elapsed(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, typeof(*dev_priv), + gpu_error.hangcheck_work.work); + struct intel_engine_cs *engine; + enum intel_engine_id id; + unsigned int hung = 0, stuck = 0; + int busy_count = 0; +#define BUSY 1 +#define KICK 5 +#define HUNG 20 +#define ACTIVE_DECAY 15 + + if (!i915.enable_hangcheck) + return; + + if (!READ_ONCE(dev_priv->gt.awake)) + return; + + /* As enabling the GPU requires fairly extensive mmio access, + * periodically arm the mmio checker to see if we are triggering + * any invalid access. + */ + intel_uncore_arm_unclaimed_mmio_detection(dev_priv); + + for_each_engine(engine, dev_priv, id) { + bool busy = intel_engine_has_waiter(engine); + u64 acthd; + u32 seqno; + u32 submit; + + semaphore_clear_deadlocks(dev_priv); + + /* We don't strictly need an irq-barrier here, as we are not + * serving an interrupt request, be paranoid in case the + * barrier has side-effects (such as preventing a broken + * cacheline snoop) and so be sure that we can see the seqno + * advance. If the seqno should stick, due to a stale + * cacheline, we would erroneously declare the GPU hung. + */ + if (engine->irq_seqno_barrier) + engine->irq_seqno_barrier(engine); + + acthd = intel_engine_get_active_head(engine); + seqno = intel_engine_get_seqno(engine); + submit = intel_engine_last_submit(engine); + + if (engine->hangcheck.seqno == seqno) { + if (i915_seqno_passed(seqno, submit)) { + engine->hangcheck.action = HANGCHECK_IDLE; + } else { + /* We always increment the hangcheck score + * if the engine is busy and still processing + * the same request, so that no single request + * can run indefinitely (such as a chain of + * batches). The only time we do not increment + * the hangcheck score on this ring, if this + * engine is in a legitimate wait for another + * engine. In that case the waiting engine is a + * victim and we want to be sure we catch the + * right culprit. Then every time we do kick + * the ring, add a small increment to the + * score so that we can catch a batch that is + * being repeatedly kicked and so responsible + * for stalling the machine. + */ + engine->hangcheck.action = + engine_stuck(engine, acthd); + + switch (engine->hangcheck.action) { + case HANGCHECK_IDLE: + case HANGCHECK_WAIT: + break; + case HANGCHECK_ACTIVE: + engine->hangcheck.score += BUSY; + break; + case HANGCHECK_KICK: + engine->hangcheck.score += KICK; + break; + case HANGCHECK_HUNG: + engine->hangcheck.score += HUNG; + break; + } + } + + if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { + hung |= intel_engine_flag(engine); + if (engine->hangcheck.action != HANGCHECK_HUNG) + stuck |= intel_engine_flag(engine); + } + } else { + engine->hangcheck.action = HANGCHECK_ACTIVE; + + /* Gradually reduce the count so that we catch DoS + * attempts across multiple batches. + */ + if (engine->hangcheck.score > 0) + engine->hangcheck.score -= ACTIVE_DECAY; + if (engine->hangcheck.score < 0) + engine->hangcheck.score = 0; + + /* Clear head and subunit states on seqno movement */ + acthd = 0; + + memset(&engine->hangcheck.instdone, 0, + sizeof(engine->hangcheck.instdone)); + } + + engine->hangcheck.seqno = seqno; + engine->hangcheck.acthd = acthd; + busy_count += busy; + } + + if (hung) { + char msg[80]; + unsigned int tmp; + int len; + + /* If some rings hung but others were still busy, only + * blame the hanging rings in the synopsis. + */ + if (stuck != hung) + hung &= ~stuck; + len = scnprintf(msg, sizeof(msg), + "%s on ", stuck == hung ? "No progress" : "Hang"); + for_each_engine_masked(engine, dev_priv, hung, tmp) + len += scnprintf(msg + len, sizeof(msg) - len, + "%s, ", engine->name); + msg[len-2] = '\0'; + + return i915_handle_error(dev_priv, hung, msg); + } + + /* Reset timer in case GPU hangs without another request being added */ + if (busy_count) + i915_queue_hangcheck(dev_priv); +} + +void intel_engine_init_hangcheck(struct intel_engine_cs *engine) +{ + memset(&engine->hangcheck, 0, sizeof(engine->hangcheck)); +} + +void intel_hangcheck_init(struct drm_i915_private *i915) +{ + INIT_DELAYED_WORK(&i915->gpu_error.hangcheck_work, + i915_hangcheck_elapsed); +} diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index af8715f31807..fb88e32e25a3 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -975,14 +975,16 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, pipe_config->lane_count = 4; } -static void intel_enable_hdmi_audio(struct intel_encoder *encoder) +static void intel_enable_hdmi_audio(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); WARN_ON(!crtc->config->has_hdmi_sink); DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n", pipe_name(crtc->pipe)); - intel_audio_codec_enable(encoder); + intel_audio_codec_enable(encoder, pipe_config, conn_state); } static void g4x_enable_hdmi(struct intel_encoder *encoder, @@ -991,21 +993,20 @@ static void g4x_enable_hdmi(struct intel_encoder *encoder, { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); u32 temp; temp = I915_READ(intel_hdmi->hdmi_reg); temp |= SDVO_ENABLE; - if (crtc->config->has_audio) + if (pipe_config->has_audio) temp |= SDVO_AUDIO_ENABLE; I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - if (crtc->config->has_audio) - intel_enable_hdmi_audio(encoder); + if (pipe_config->has_audio) + intel_enable_hdmi_audio(encoder, pipe_config, conn_state); } static void ibx_enable_hdmi(struct intel_encoder *encoder, @@ -1040,8 +1041,8 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder, * FIXME: BSpec says this should be done at the end of * of the modeset sequence, so not sure if this isn't too soon. */ - if (crtc->config->pipe_bpp > 24 && - crtc->config->pixel_multiplier > 1) { + if (pipe_config->pipe_bpp > 24 && + pipe_config->pixel_multiplier > 1) { I915_WRITE(intel_hdmi->hdmi_reg, temp & ~SDVO_ENABLE); POSTING_READ(intel_hdmi->hdmi_reg); @@ -1055,8 +1056,8 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder, POSTING_READ(intel_hdmi->hdmi_reg); } - if (crtc->config->has_audio) - intel_enable_hdmi_audio(encoder); + if (pipe_config->has_audio) + intel_enable_hdmi_audio(encoder, pipe_config, conn_state); } static void cpt_enable_hdmi(struct intel_encoder *encoder, @@ -1073,7 +1074,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder, temp = I915_READ(intel_hdmi->hdmi_reg); temp |= SDVO_ENABLE; - if (crtc->config->has_audio) + if (pipe_config->has_audio) temp |= SDVO_AUDIO_ENABLE; /* @@ -1086,7 +1087,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder, * 4. enable HDMI clock gating */ - if (crtc->config->pipe_bpp > 24) { + if (pipe_config->pipe_bpp > 24) { I915_WRITE(TRANS_CHICKEN1(pipe), I915_READ(TRANS_CHICKEN1(pipe)) | TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE); @@ -1098,7 +1099,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder, I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - if (crtc->config->pipe_bpp > 24) { + if (pipe_config->pipe_bpp > 24) { temp &= ~SDVO_COLOR_FORMAT_MASK; temp |= HDMI_COLOR_FORMAT_12bpc; @@ -1110,8 +1111,8 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder, ~TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE); } - if (crtc->config->has_audio) - intel_enable_hdmi_audio(encoder); + if (pipe_config->has_audio) + intel_enable_hdmi_audio(encoder, pipe_config, conn_state); } static void vlv_enable_hdmi(struct intel_encoder *encoder, @@ -1164,7 +1165,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder, I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -1178,9 +1179,7 @@ static void g4x_disable_hdmi(struct intel_encoder *encoder, struct intel_crtc_state *old_crtc_state, struct drm_connector_state *old_conn_state) { - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - - if (crtc->config->has_audio) + if (old_crtc_state->has_audio) intel_audio_codec_disable(encoder); intel_disable_hdmi(encoder, old_crtc_state, old_conn_state); @@ -1190,9 +1189,7 @@ static void pch_disable_hdmi(struct intel_encoder *encoder, struct intel_crtc_state *old_crtc_state, struct drm_connector_state *old_conn_state) { - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - - if (crtc->config->has_audio) + if (old_crtc_state->has_audio) intel_audio_codec_disable(encoder); } @@ -1645,13 +1642,12 @@ static void intel_hdmi_pre_enable(struct intel_encoder *encoder, struct drm_connector_state *conn_state) { struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; intel_hdmi_prepare(encoder); intel_hdmi->set_infoframes(&encoder->base, - intel_crtc->config->has_hdmi_sink, + pipe_config->has_hdmi_sink, adjusted_mode); } @@ -1663,9 +1659,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder, struct intel_hdmi *intel_hdmi = &dport->hdmi; struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; vlv_phy_pre_encoder_enable(encoder); @@ -1674,7 +1668,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder, 0x2b247878); intel_hdmi->set_infoframes(&encoder->base, - intel_crtc->config->has_hdmi_sink, + pipe_config->has_hdmi_sink, adjusted_mode); g4x_enable_hdmi(encoder, pipe_config, conn_state); diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c index 334d47b5811a..3d546c019de0 100644 --- a/drivers/gpu/drm/i915/intel_hotplug.c +++ b/drivers/gpu/drm/i915/intel_hotplug.c @@ -501,7 +501,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work) if (intel_connector->mst_port) continue; - if (!connector->polled && I915_HAS_HOTPLUG(dev) && + if (!connector->polled && I915_HAS_HOTPLUG(dev_priv) && intel_connector->encoder->hpd_pin > HPD_NONE) { connector->polled = enabled ? DRM_CONNECTOR_POLL_CONNECT | diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index bc86585b9fbb..0a09024d6ca3 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -365,7 +365,7 @@ static u64 execlists_update_context(struct drm_i915_gem_request *rq) struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt; u32 *reg_state = ce->lrc_reg_state; - reg_state[CTX_RING_TAIL+1] = intel_ring_offset(rq->ring, rq->tail); + reg_state[CTX_RING_TAIL+1] = rq->tail; /* True 32b PPGTT with dynamic page allocation: update PDP * registers and point the unallocated PDPs to scratch page. @@ -432,15 +432,17 @@ static bool can_merge_ctx(const struct i915_gem_context *prev, static void execlists_dequeue(struct intel_engine_cs *engine) { - struct drm_i915_gem_request *cursor, *last; + struct drm_i915_gem_request *last; struct execlist_port *port = engine->execlist_port; + unsigned long flags; + struct rb_node *rb; bool submit = false; last = port->request; if (last) /* WaIdleLiteRestore:bdw,skl * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL - * as we resubmit the request. See gen8_emit_request() + * as we resubmit the request. See gen8_emit_breadcrumb() * for where we prepare the padding after the end of the * request. */ @@ -469,8 +471,12 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * and context switches) submission. */ - spin_lock(&engine->execlist_lock); - list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) { + spin_lock_irqsave(&engine->timeline->lock, flags); + rb = engine->execlist_first; + while (rb) { + struct drm_i915_gem_request *cursor = + rb_entry(rb, typeof(*cursor), priotree.node); + /* Can we combine this request with the current port? It has to * be the same context/ringbuffer and not have any exceptions * (e.g. GVT saying never to combine contexts). @@ -493,7 +499,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * context (even though a different request) to * the second port. */ - if (ctx_single_port_submission(cursor->ctx)) + if (ctx_single_port_submission(last->ctx) || + ctx_single_port_submission(cursor->ctx)) break; GEM_BUG_ON(last->ctx == cursor->ctx); @@ -501,17 +508,30 @@ static void execlists_dequeue(struct intel_engine_cs *engine) i915_gem_request_assign(&port->request, last); port++; } + + rb = rb_next(rb); + rb_erase(&cursor->priotree.node, &engine->execlist_queue); + RB_CLEAR_NODE(&cursor->priotree.node); + cursor->priotree.priority = INT_MAX; + + /* We keep the previous context alive until we retire the + * following request. This ensures that any the context object + * is still pinned for any residual writes the HW makes into it + * on the context switch into the next object following the + * breadcrumb. Otherwise, we may retire the context too early. + */ + cursor->previous_context = engine->last_context; + engine->last_context = cursor->ctx; + + __i915_gem_request_submit(cursor); last = cursor; submit = true; } if (submit) { - /* Decouple all the requests submitted from the queue */ - engine->execlist_queue.next = &cursor->execlist_link; - cursor->execlist_link.prev = &engine->execlist_queue; - i915_gem_request_assign(&port->request, last); + engine->execlist_first = rb; } - spin_unlock(&engine->execlist_lock); + spin_unlock_irqrestore(&engine->timeline->lock, flags); if (submit) execlists_submit_ports(engine); @@ -522,6 +542,28 @@ static bool execlists_elsp_idle(struct intel_engine_cs *engine) return !engine->execlist_port[0].request; } +/** + * intel_execlists_idle() - Determine if all engine submission ports are idle + * @dev_priv: i915 device private + * + * Return true if there are no requests pending on any of the submission ports + * of any engines. + */ +bool intel_execlists_idle(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + if (!i915.enable_execlists) + return true; + + for_each_engine(engine, dev_priv, id) + if (!execlists_elsp_idle(engine)) + return false; + + return true; +} + static bool execlists_elsp_ready(struct intel_engine_cs *engine) { int port; @@ -592,18 +634,147 @@ static void intel_lrc_irq_handler(unsigned long data) intel_uncore_forcewake_put(dev_priv, engine->fw_domains); } +static bool insert_request(struct i915_priotree *pt, struct rb_root *root) +{ + struct rb_node **p, *rb; + bool first = true; + + /* most positive priority is scheduled first, equal priorities fifo */ + rb = NULL; + p = &root->rb_node; + while (*p) { + struct i915_priotree *pos; + + rb = *p; + pos = rb_entry(rb, typeof(*pos), node); + if (pt->priority > pos->priority) { + p = &rb->rb_left; + } else { + p = &rb->rb_right; + first = false; + } + } + rb_link_node(&pt->node, rb, p); + rb_insert_color(&pt->node, root); + + return first; +} + static void execlists_submit_request(struct drm_i915_gem_request *request) { struct intel_engine_cs *engine = request->engine; unsigned long flags; - spin_lock_irqsave(&engine->execlist_lock, flags); + /* Will be called from irq-context when using foreign fences. */ + spin_lock_irqsave(&engine->timeline->lock, flags); - list_add_tail(&request->execlist_link, &engine->execlist_queue); + if (insert_request(&request->priotree, &engine->execlist_queue)) + engine->execlist_first = &request->priotree.node; if (execlists_elsp_idle(engine)) tasklet_hi_schedule(&engine->irq_tasklet); - spin_unlock_irqrestore(&engine->execlist_lock, flags); + spin_unlock_irqrestore(&engine->timeline->lock, flags); +} + +static struct intel_engine_cs * +pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked) +{ + struct intel_engine_cs *engine; + + engine = container_of(pt, + struct drm_i915_gem_request, + priotree)->engine; + if (engine != locked) { + if (locked) + spin_unlock_irq(&locked->timeline->lock); + spin_lock_irq(&engine->timeline->lock); + } + + return engine; +} + +static void execlists_schedule(struct drm_i915_gem_request *request, int prio) +{ + static DEFINE_MUTEX(lock); + struct intel_engine_cs *engine = NULL; + struct i915_dependency *dep, *p; + struct i915_dependency stack; + LIST_HEAD(dfs); + + if (prio <= READ_ONCE(request->priotree.priority)) + return; + + /* Need global lock to use the temporary link inside i915_dependency */ + mutex_lock(&lock); + + stack.signaler = &request->priotree; + list_add(&stack.dfs_link, &dfs); + + /* Recursively bump all dependent priorities to match the new request. + * + * A naive approach would be to use recursion: + * static void update_priorities(struct i915_priotree *pt, prio) { + * list_for_each_entry(dep, &pt->signalers_list, signal_link) + * update_priorities(dep->signal, prio) + * insert_request(pt); + * } + * but that may have unlimited recursion depth and so runs a very + * real risk of overunning the kernel stack. Instead, we build + * a flat list of all dependencies starting with the current request. + * As we walk the list of dependencies, we add all of its dependencies + * to the end of the list (this may include an already visited + * request) and continue to walk onwards onto the new dependencies. The + * end result is a topological list of requests in reverse order, the + * last element in the list is the request we must execute first. + */ + list_for_each_entry_safe(dep, p, &dfs, dfs_link) { + struct i915_priotree *pt = dep->signaler; + + list_for_each_entry(p, &pt->signalers_list, signal_link) + if (prio > READ_ONCE(p->signaler->priority)) + list_move_tail(&p->dfs_link, &dfs); + + p = list_next_entry(dep, dfs_link); + if (!RB_EMPTY_NODE(&pt->node)) + continue; + + engine = pt_lock_engine(pt, engine); + + /* If it is not already in the rbtree, we can update the + * priority inplace and skip over it (and its dependencies) + * if it is referenced *again* as we descend the dfs. + */ + if (prio > pt->priority && RB_EMPTY_NODE(&pt->node)) { + pt->priority = prio; + list_del_init(&dep->dfs_link); + } + } + + /* Fifo and depth-first replacement ensure our deps execute before us */ + list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { + struct i915_priotree *pt = dep->signaler; + + INIT_LIST_HEAD(&dep->dfs_link); + + engine = pt_lock_engine(pt, engine); + + if (prio <= pt->priority) + continue; + + GEM_BUG_ON(RB_EMPTY_NODE(&pt->node)); + + pt->priority = prio; + rb_erase(&pt->node, &engine->execlist_queue); + if (insert_request(pt, &engine->execlist_queue)) + engine->execlist_first = &pt->node; + } + + if (engine) + spin_unlock_irq(&engine->timeline->lock); + + mutex_unlock(&lock); + + /* XXX Do we need to preempt to make room for us and our deps? */ } int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request) @@ -671,46 +842,6 @@ err_unpin: return ret; } -/* - * intel_logical_ring_advance() - advance the tail and prepare for submission - * @request: Request to advance the logical ringbuffer of. - * - * The tail is updated in our logical ringbuffer struct, not in the actual context. What - * really happens during submission is that the context and current tail will be placed - * on a queue waiting for the ELSP to be ready to accept a new context submission. At that - * point, the tail *inside* the context is updated and the ELSP written to. - */ -static int -intel_logical_ring_advance(struct drm_i915_gem_request *request) -{ - struct intel_ring *ring = request->ring; - struct intel_engine_cs *engine = request->engine; - - intel_ring_advance(ring); - request->tail = ring->tail; - - /* - * Here we add two extra NOOPs as padding to avoid - * lite restore of a context with HEAD==TAIL. - * - * Caller must reserve WA_TAIL_DWORDS for us! - */ - intel_ring_emit(ring, MI_NOOP); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); - request->wa_tail = ring->tail; - - /* We keep the previous context alive until we retire the following - * request. This ensures that any the context object is still pinned - * for any residual writes the HW makes into it on the context switch - * into the next object following the breadcrumb. Otherwise, we may - * retire the context too early. - */ - request->previous_context = engine->last_context; - engine->last_context = request->ctx; - return 0; -} - static int intel_lr_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { @@ -744,7 +875,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx, ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = i915_ggtt_offset(ce->ring->vma); - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true; /* Invalidate GuC TLB. */ if (i915.enable_guc_submission) { @@ -1566,39 +1697,35 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) * used as a workaround for not being allowed to do lite * restore with HEAD==TAIL (WaIdleLiteRestore). */ - -static int gen8_emit_request(struct drm_i915_gem_request *request) +static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *out) { - struct intel_ring *ring = request->ring; - int ret; - - ret = intel_ring_begin(request, 6 + WA_TAIL_DWORDS); - if (ret) - return ret; + *out++ = MI_NOOP; + *out++ = MI_NOOP; + request->wa_tail = intel_ring_offset(request->ring, out); +} +static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, + u32 *out) +{ /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */ BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5)); - intel_ring_emit(ring, (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); - intel_ring_emit(ring, - intel_hws_seqno_address(request->engine) | - MI_FLUSH_DW_USE_GTT); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, request->fence.seqno); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_emit(ring, MI_NOOP); - return intel_logical_ring_advance(request); -} + *out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW; + *out++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT; + *out++ = 0; + *out++ = request->global_seqno; + *out++ = MI_USER_INTERRUPT; + *out++ = MI_NOOP; + request->tail = intel_ring_offset(request->ring, out); -static int gen8_emit_request_render(struct drm_i915_gem_request *request) -{ - struct intel_ring *ring = request->ring; - int ret; + gen8_emit_wa_tail(request, out); +} - ret = intel_ring_begin(request, 8 + WA_TAIL_DWORDS); - if (ret) - return ret; +static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS; +static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request, + u32 *out) +{ /* We're using qword write, seqno should be aligned to 8 bytes. */ BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1); @@ -1606,21 +1733,24 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request) * need a prior CS_STALL, which is emitted by the flush * following the batch. */ - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(ring, - (PIPE_CONTROL_GLOBAL_GTT_IVB | - PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_QW_WRITE)); - intel_ring_emit(ring, intel_hws_seqno_address(request->engine)); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(request)); + *out++ = GFX_OP_PIPE_CONTROL(6); + *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_QW_WRITE); + *out++ = intel_hws_seqno_address(request->engine); + *out++ = 0; + *out++ = request->global_seqno; /* We're thrashing one dword of HWS. */ - intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_emit(ring, MI_NOOP); - return intel_logical_ring_advance(request); + *out++ = 0; + *out++ = MI_USER_INTERRUPT; + *out++ = MI_NOOP; + request->tail = intel_ring_offset(request->ring, out); + + gen8_emit_wa_tail(request, out); } +static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS; + static int gen8_init_rcs_context(struct drm_i915_gem_request *req) { int ret; @@ -1637,7 +1767,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req) if (ret) DRM_ERROR("MOCS failed to program: expect performance issues.\n"); - return i915_gem_render_state_init(req); + return i915_gem_render_state_emit(req); } /** @@ -1683,8 +1813,10 @@ void intel_execlists_enable_submission(struct drm_i915_private *dev_priv) struct intel_engine_cs *engine; enum intel_engine_id id; - for_each_engine(engine, dev_priv, id) + for_each_engine(engine, dev_priv, id) { engine->submit_request = execlists_submit_request; + engine->schedule = execlists_schedule; + } } static void @@ -1694,8 +1826,10 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->init_hw = gen8_init_common_ring; engine->reset_hw = reset_common_ring; engine->emit_flush = gen8_emit_flush; - engine->emit_request = gen8_emit_request; + engine->emit_breadcrumb = gen8_emit_breadcrumb; + engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz; engine->submit_request = execlists_submit_request; + engine->schedule = execlists_schedule; engine->irq_enable = gen8_logical_ring_enable_irq; engine->irq_disable = gen8_logical_ring_disable_irq; @@ -1816,7 +1950,8 @@ int logical_render_ring_init(struct intel_engine_cs *engine) engine->init_hw = gen8_init_render_ring; engine->init_context = gen8_init_rcs_context; engine->emit_flush = gen8_emit_flush_render; - engine->emit_request = gen8_emit_request_render; + engine->emit_breadcrumb = gen8_emit_breadcrumb_render; + engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz; ret = intel_engine_create_scratch(engine, 4096); if (ret) @@ -2042,7 +2177,7 @@ populate_lr_context(struct i915_gem_context *ctx, DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret); return ret; } - ctx_obj->dirty = true; + ctx_obj->mm.dirty = true; /* The second page of the context object contains some fields which must * be set up prior to the first execution. */ @@ -2180,7 +2315,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv) reg[CTX_RING_HEAD+1] = 0; reg[CTX_RING_TAIL+1] = 0; - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true; i915_gem_object_unpin_map(ce->state->obj); ce->ring->head = ce->ring->tail = 0; diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h index 4fed8165f98a..c1f546180ba2 100644 --- a/drivers/gpu/drm/i915/intel_lrc.h +++ b/drivers/gpu/drm/i915/intel_lrc.h @@ -95,5 +95,6 @@ uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx, int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists); void intel_execlists_enable_submission(struct drm_i915_private *dev_priv); +bool intel_execlists_idle(struct drm_i915_private *dev_priv); #endif /* _INTEL_LRC_H_ */ diff --git a/drivers/gpu/drm/i915/intel_lspcon.c b/drivers/gpu/drm/i915/intel_lspcon.c index 632149c6b3ad..daa523410953 100644 --- a/drivers/gpu/drm/i915/intel_lspcon.c +++ b/drivers/gpu/drm/i915/intel_lspcon.c @@ -27,10 +27,18 @@ #include <drm/drm_dp_dual_mode_helper.h> #include "intel_drv.h" +static struct intel_dp *lspcon_to_intel_dp(struct intel_lspcon *lspcon) +{ + struct intel_digital_port *dig_port = + container_of(lspcon, struct intel_digital_port, lspcon); + + return &dig_port->dp; +} + static enum drm_lspcon_mode lspcon_get_current_mode(struct intel_lspcon *lspcon) { enum drm_lspcon_mode current_mode = DRM_LSPCON_MODE_INVALID; - struct i2c_adapter *adapter = &lspcon->aux->ddc; + struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc; if (drm_lspcon_get_mode(adapter, ¤t_mode)) DRM_ERROR("Error reading LSPCON mode\n"); @@ -45,7 +53,7 @@ static int lspcon_change_mode(struct intel_lspcon *lspcon, { int err; enum drm_lspcon_mode current_mode; - struct i2c_adapter *adapter = &lspcon->aux->ddc; + struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc; err = drm_lspcon_get_mode(adapter, ¤t_mode); if (err) { @@ -72,7 +80,7 @@ static int lspcon_change_mode(struct intel_lspcon *lspcon, static bool lspcon_probe(struct intel_lspcon *lspcon) { enum drm_dp_dual_mode_type adaptor_type; - struct i2c_adapter *adapter = &lspcon->aux->ddc; + struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc; /* Lets probe the adaptor and check its type */ adaptor_type = drm_dp_dual_mode_detect(adapter); @@ -89,8 +97,43 @@ static bool lspcon_probe(struct intel_lspcon *lspcon) return true; } +static void lspcon_resume_in_pcon_wa(struct intel_lspcon *lspcon) +{ + struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + unsigned long start = jiffies; + + if (!lspcon->desc_valid) + return; + + while (1) { + struct intel_dp_desc desc; + + /* + * The w/a only applies in PCON mode and we don't expect any + * AUX errors. + */ + if (!__intel_dp_read_desc(intel_dp, &desc)) + return; + + if (!memcmp(&intel_dp->desc, &desc, sizeof(desc))) { + DRM_DEBUG_KMS("LSPCON recovering in PCON mode after %u ms\n", + jiffies_to_msecs(jiffies - start)); + return; + } + + if (time_after(jiffies, start + msecs_to_jiffies(1000))) + break; + + usleep_range(10000, 15000); + } + + DRM_DEBUG_KMS("LSPCON DP descriptor mismatch after resume\n"); +} + void lspcon_resume(struct intel_lspcon *lspcon) { + lspcon_resume_in_pcon_wa(lspcon); + if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON, true)) DRM_ERROR("LSPCON resume failed\n"); else @@ -111,7 +154,6 @@ bool lspcon_init(struct intel_digital_port *intel_dig_port) lspcon->active = false; lspcon->mode = DRM_LSPCON_MODE_INVALID; - lspcon->aux = &dp->aux; if (!lspcon_probe(lspcon)) { DRM_ERROR("Failed to probe lspcon\n"); @@ -131,6 +173,13 @@ bool lspcon_init(struct intel_digital_port *intel_dig_port) } } + if (!intel_dp_read_dpcd(dp)) { + DRM_ERROR("LSPCON DPCD read failed\n"); + return false; + } + + lspcon->desc_valid = intel_dp_read_desc(dp); + DRM_DEBUG_KMS("Success: LSPCON init\n"); return true; } diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 199b90c7907a..d12ef0047d49 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -122,8 +122,7 @@ out: static void intel_lvds_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); u32 tmp, flags = 0; @@ -139,12 +138,12 @@ static void intel_lvds_get_config(struct intel_encoder *encoder, pipe_config->base.adjusted_mode.flags |= flags; - if (INTEL_INFO(dev)->gen < 5) + if (INTEL_GEN(dev_priv) < 5) pipe_config->gmch_pfit.lvds_border_bits = tmp & LVDS_BORDER_ENABLE; /* gen2/3 store dither state in pfit control, needs to match */ - if (INTEL_INFO(dev)->gen < 4) { + if (INTEL_GEN(dev_priv) < 4) { tmp = I915_READ(PFIT_CONTROL); pipe_config->gmch_pfit.control |= tmp & PANEL_8TO6_DITHER_ENABLE; @@ -985,7 +984,7 @@ void intel_lvds_init(struct drm_device *dev) struct drm_display_mode *fixed_mode = NULL; struct drm_display_mode *downclock_mode = NULL; struct edid *edid; - struct drm_crtc *crtc; + struct intel_crtc *crtc; i915_reg_t lvds_reg; u32 lvds; int pipe; @@ -1163,10 +1162,10 @@ void intel_lvds_init(struct drm_device *dev) goto failed; pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0; - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc && (lvds & LVDS_PORT_EN)) { - fixed_mode = intel_crtc_mode_get(dev, crtc); + fixed_mode = intel_crtc_mode_get(dev, &crtc->base); if (fixed_mode) { DRM_DEBUG_KMS("using current (BIOS) mode: "); drm_mode_debug_printmodeline(fixed_mode); diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index 7acbbbf97833..f4429f67a4e3 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -642,24 +642,6 @@ static struct notifier_block intel_opregion_notifier = { * (version 3) */ -static u32 get_did(struct intel_opregion *opregion, int i) -{ - u32 did; - - if (i < ARRAY_SIZE(opregion->acpi->didl)) { - did = opregion->acpi->didl[i]; - } else { - i -= ARRAY_SIZE(opregion->acpi->didl); - - if (WARN_ON(i >= ARRAY_SIZE(opregion->acpi->did2))) - return 0; - - did = opregion->acpi->did2[i]; - } - - return did; -} - static void set_did(struct intel_opregion *opregion, int i, u32 val) { if (i < ARRAY_SIZE(opregion->acpi->didl)) { @@ -674,11 +656,11 @@ static void set_did(struct intel_opregion *opregion, int i, u32 val) } } -static u32 acpi_display_type(struct drm_connector *connector) +static u32 acpi_display_type(struct intel_connector *connector) { u32 display_type; - switch (connector->connector_type) { + switch (connector->base.connector_type) { case DRM_MODE_CONNECTOR_VGA: case DRM_MODE_CONNECTOR_DVIA: display_type = ACPI_DISPLAY_TYPE_VGA; @@ -707,7 +689,7 @@ static u32 acpi_display_type(struct drm_connector *connector) display_type = ACPI_DISPLAY_TYPE_OTHER; break; default: - MISSING_CASE(connector->connector_type); + MISSING_CASE(connector->base.connector_type); display_type = ACPI_DISPLAY_TYPE_OTHER; break; } @@ -718,34 +700,9 @@ static u32 acpi_display_type(struct drm_connector *connector) static void intel_didl_outputs(struct drm_i915_private *dev_priv) { struct intel_opregion *opregion = &dev_priv->opregion; - struct pci_dev *pdev = dev_priv->drm.pdev; - struct drm_connector *connector; - acpi_handle handle; - struct acpi_device *acpi_dev, *acpi_cdev, *acpi_video_bus = NULL; - unsigned long long device_id; - acpi_status status; - u32 temp, max_outputs; - int i = 0; - - handle = ACPI_HANDLE(&pdev->dev); - if (!handle || acpi_bus_get_device(handle, &acpi_dev)) - return; - - if (acpi_is_video_device(handle)) - acpi_video_bus = acpi_dev; - else { - list_for_each_entry(acpi_cdev, &acpi_dev->children, node) { - if (acpi_is_video_device(acpi_cdev->handle)) { - acpi_video_bus = acpi_cdev; - break; - } - } - } - - if (!acpi_video_bus) { - DRM_DEBUG_KMS("No ACPI video bus found\n"); - return; - } + struct intel_connector *connector; + int i = 0, max_outputs; + int display_index[16] = {}; /* * In theory, did2, the extended didl, gets added at opregion version @@ -757,64 +714,58 @@ static void intel_didl_outputs(struct drm_i915_private *dev_priv) max_outputs = ARRAY_SIZE(opregion->acpi->didl) + ARRAY_SIZE(opregion->acpi->did2); - list_for_each_entry(acpi_cdev, &acpi_video_bus->children, node) { - if (i >= max_outputs) { - DRM_DEBUG_KMS("More than %u outputs detected via ACPI\n", - max_outputs); - return; - } - status = acpi_evaluate_integer(acpi_cdev->handle, "_ADR", - NULL, &device_id); - if (ACPI_SUCCESS(status)) { - if (!device_id) - goto blind_set; - set_did(opregion, i++, (u32)(device_id & 0x0f0f)); - } + for_each_intel_connector(&dev_priv->drm, connector) { + u32 device_id, type; + + device_id = acpi_display_type(connector); + + /* Use display type specific display index. */ + type = (device_id & ACPI_DISPLAY_TYPE_MASK) + >> ACPI_DISPLAY_TYPE_SHIFT; + device_id |= display_index[type]++ << ACPI_DISPLAY_INDEX_SHIFT; + + connector->acpi_device_id = device_id; + if (i < max_outputs) + set_did(opregion, i, device_id); + i++; } -end: DRM_DEBUG_KMS("%d outputs detected\n", i); + if (i > max_outputs) + DRM_ERROR("More than %d outputs in connector list\n", + max_outputs); + /* If fewer than max outputs, the list must be null terminated */ if (i < max_outputs) set_did(opregion, i, 0); - return; - -blind_set: - i = 0; - list_for_each_entry(connector, - &dev_priv->drm.mode_config.connector_list, head) { - int display_type = acpi_display_type(connector); - - if (i >= max_outputs) { - DRM_DEBUG_KMS("More than %u outputs in connector list\n", - max_outputs); - return; - } - - temp = get_did(opregion, i); - set_did(opregion, i, temp | (1 << 31) | display_type | i); - i++; - } - goto end; } static void intel_setup_cadls(struct drm_i915_private *dev_priv) { struct intel_opregion *opregion = &dev_priv->opregion; + struct intel_connector *connector; int i = 0; - u32 disp_id; - - /* Initialize the CADL field by duplicating the DIDL values. - * Technically, this is not always correct as display outputs may exist, - * but not active. This initialization is necessary for some Clevo - * laptops that check this field before processing the brightness and - * display switching hotkeys. Just like DIDL, CADL is NULL-terminated if - * there are less than eight devices. */ - do { - disp_id = get_did(opregion, i); - opregion->acpi->cadl[i] = disp_id; - } while (++i < 8 && disp_id != 0); + + /* + * Initialize the CADL field from the connector device ids. This is + * essentially the same as copying from the DIDL. Technically, this is + * not always correct as display outputs may exist, but not active. This + * initialization is necessary for some Clevo laptops that check this + * field before processing the brightness and display switching hotkeys. + * + * Note that internal panels should be at the front of the connector + * list already, ensuring they're not left out. + */ + for_each_intel_connector(&dev_priv->drm, connector) { + if (i >= ARRAY_SIZE(opregion->acpi->cadl)) + break; + opregion->acpi->cadl[i++] = connector->acpi_device_id; + } + + /* If fewer than 8 active devices, the list must be null terminated */ + if (i < ARRAY_SIZE(opregion->acpi->cadl)) + opregion->acpi->cadl[i] = 0; } void intel_opregion_register(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index 25bcd4a178d3..fd0e4dac7cc1 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -1222,7 +1222,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, out_unlock: mutex_unlock(&dev->struct_mutex); drm_modeset_unlock_all(dev); - i915_gem_object_put_unlocked(new_bo); + i915_gem_object_put(new_bo); out_free: kfree(params); @@ -1466,7 +1466,7 @@ void intel_cleanup_overlay(struct drm_i915_private *dev_priv) * hardware should be off already */ WARN_ON(dev_priv->overlay->active); - i915_gem_object_put_unlocked(dev_priv->overlay->reg_bo); + i915_gem_object_put(dev_priv->overlay->reg_bo); kfree(dev_priv->overlay); } diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index be4b4d546fd9..08ab6d762ca4 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -304,7 +304,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, struct intel_crtc_state *pipe_config, int fitting_mode) { - struct drm_device *dev = intel_crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0; struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; @@ -325,7 +325,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, break; case DRM_MODE_SCALE_ASPECT: /* Scale but preserve the aspect ratio */ - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) i965_scale_aspect(pipe_config, &pfit_control); else i9xx_scale_aspect(pipe_config, &pfit_control, @@ -339,7 +339,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay || pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) { pfit_control |= PFIT_ENABLE; - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) pfit_control |= PFIT_SCALING_AUTO; else pfit_control |= (VERT_AUTO_SCALE | @@ -355,7 +355,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, /* 965+ wants fuzzy fitting */ /* FIXME: handle multiple panels by failing gracefully */ - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) | PFIT_FILTER_FUZZY); @@ -366,7 +366,7 @@ out: } /* Make sure pre-965 set dither correctly for 18bpp panels. */ - if (INTEL_INFO(dev)->gen < 4 && pipe_config->pipe_bpp == 18) + if (INTEL_GEN(dev_priv) < 4 && pipe_config->pipe_bpp == 18) pfit_control |= PANEL_8TO6_DITHER_ENABLE; pipe_config->gmch_pfit.control = pfit_control; @@ -1722,7 +1722,7 @@ int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe) DRM_DEBUG_KMS("Connector %s backlight initialized, %s, brightness %u/%u\n", connector->name, - panel->backlight.enabled ? "enabled" : "disabled", + enableddisabled(panel->backlight.enabled), panel->backlight.level, panel->backlight.max); return 0; diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 560fc7af8267..d67974eb127a 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -31,6 +31,7 @@ #include "intel_drv.h" #include "../../../platform/x86/intel_ips.h" #include <linux/module.h> +#include <drm/drm_atomic_helper.h> /** * DOC: RC6 @@ -55,10 +56,8 @@ #define INTEL_RC6p_ENABLE (1<<1) #define INTEL_RC6pp_ENABLE (1<<2) -static void gen9_init_clock_gating(struct drm_device *dev) +static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl */ I915_WRITE(CHICKEN_PAR1_1, I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); @@ -81,11 +80,9 @@ static void gen9_init_clock_gating(struct drm_device *dev) ILK_DPFC_DISABLE_DUMMY0); } -static void bxt_init_clock_gating(struct drm_device *dev) +static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - gen9_init_clock_gating(dev); + gen9_init_clock_gating(dev_priv); /* WaDisableSDEUnitClockGating:bxt */ I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | @@ -107,9 +104,8 @@ static void bxt_init_clock_gating(struct drm_device *dev) PWM1_GATING_DIS | PWM2_GATING_DIS); } -static void i915_pineview_get_mem_freq(struct drm_device *dev) +static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 tmp; tmp = I915_READ(CLKCFG); @@ -146,9 +142,8 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev) dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; } -static void i915_ironlake_get_mem_freq(struct drm_device *dev) +static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u16 ddrpll, csipll; ddrpll = I915_READ16(DDRMPLL1); @@ -319,7 +314,6 @@ static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) { - struct drm_device *dev = &dev_priv->drm; u32 val; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { @@ -329,7 +323,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) } else if (IS_G4X(dev_priv) || IS_CRESTLINE(dev_priv)) { I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); POSTING_READ(FW_BLC_SELF); - } else if (IS_PINEVIEW(dev)) { + } else if (IS_PINEVIEW(dev_priv)) { val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN; val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0; I915_WRITE(DSPFW3, val); @@ -353,8 +347,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) return; } - DRM_DEBUG_KMS("memory self-refresh is %s\n", - enable ? "enabled" : "disabled"); + DRM_DEBUG_KMS("memory self-refresh is %s\n", enableddisabled(enable)); } @@ -377,10 +370,9 @@ static const int pessimal_latency_ns = 5000; #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \ ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8)) -static int vlv_get_fifo_size(struct drm_device *dev, +static int vlv_get_fifo_size(struct drm_i915_private *dev_priv, enum pipe pipe, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); int sprite0_start, sprite1_start, size; switch (pipe) { @@ -429,9 +421,8 @@ static int vlv_get_fifo_size(struct drm_device *dev, return size; } -static int i9xx_get_fifo_size(struct drm_device *dev, int plane) +static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -445,9 +436,8 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane) return size; } -static int i830_get_fifo_size(struct drm_device *dev, int plane) +static int i830_get_fifo_size(struct drm_i915_private *dev_priv, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -462,9 +452,8 @@ static int i830_get_fifo_size(struct drm_device *dev, int plane) return size; } -static int i845_get_fifo_size(struct drm_device *dev, int plane) +static int i845_get_fifo_size(struct drm_i915_private *dev_priv, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -624,11 +613,11 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz, return wm_size; } -static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) +static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv) { - struct drm_crtc *crtc, *enabled = NULL; + struct intel_crtc *crtc, *enabled = NULL; - for_each_crtc(dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { if (intel_crtc_active(crtc)) { if (enabled) return NULL; @@ -639,11 +628,10 @@ static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) return enabled; } -static void pineview_update_wm(struct drm_crtc *unused_crtc) +static void pineview_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); + struct intel_crtc *crtc; const struct cxsr_latency *latency; u32 reg; unsigned long wm; @@ -658,10 +646,13 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc) return; } - crtc = single_enabled_crtc(dev); + crtc = single_enabled_crtc(dev_priv); if (crtc) { - const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; + int cpp = drm_format_plane_cpp(fb->pixel_format, 0); int clock = adjusted_mode->crtc_clock; /* Display SR */ @@ -708,7 +699,7 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc) } } -static bool g4x_compute_wm0(struct drm_device *dev, +static bool g4x_compute_wm0(struct drm_i915_private *dev_priv, int plane, const struct intel_watermark_params *display, int display_latency_ns, @@ -717,24 +708,26 @@ static bool g4x_compute_wm0(struct drm_device *dev, int *plane_wm, int *cursor_wm) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; const struct drm_display_mode *adjusted_mode; + const struct drm_framebuffer *fb; int htotal, hdisplay, clock, cpp; int line_time_us, line_count; int entries, tlb_miss; - crtc = intel_get_crtc_for_plane(dev, plane); + crtc = intel_get_crtc_for_plane(dev_priv, plane); if (!intel_crtc_active(crtc)) { *cursor_wm = cursor->guard_size; *plane_wm = display->guard_size; return false; } - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + adjusted_mode = &crtc->config->base.adjusted_mode; + fb = crtc->base.primary->state->fb; clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; - hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; - cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + hdisplay = crtc->config->pipe_src_w; + cpp = drm_format_plane_cpp(fb->pixel_format, 0); /* Use the small buffer method to calculate plane watermark */ entries = ((clock * cpp / 1000) * display_latency_ns) / 1000; @@ -749,7 +742,7 @@ static bool g4x_compute_wm0(struct drm_device *dev, /* Use the large buffer method to calculate cursor watermark */ line_time_us = max(htotal * 1000 / clock, 1); line_count = (cursor_latency_ns / line_time_us + 1000) / 1000; - entries = line_count * crtc->cursor->state->crtc_w * cpp; + entries = line_count * crtc->base.cursor->state->crtc_w * cpp; tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8; if (tlb_miss > 0) entries += tlb_miss; @@ -768,7 +761,7 @@ static bool g4x_compute_wm0(struct drm_device *dev, * can be programmed into the associated watermark register, that watermark * must be disabled. */ -static bool g4x_check_srwm(struct drm_device *dev, +static bool g4x_check_srwm(struct drm_i915_private *dev_priv, int display_wm, int cursor_wm, const struct intel_watermark_params *display, const struct intel_watermark_params *cursor) @@ -796,15 +789,16 @@ static bool g4x_check_srwm(struct drm_device *dev, return true; } -static bool g4x_compute_srwm(struct drm_device *dev, +static bool g4x_compute_srwm(struct drm_i915_private *dev_priv, int plane, int latency_ns, const struct intel_watermark_params *display, const struct intel_watermark_params *cursor, int *display_wm, int *cursor_wm) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; const struct drm_display_mode *adjusted_mode; + const struct drm_framebuffer *fb; int hdisplay, htotal, cpp, clock; unsigned long line_time_us; int line_count, line_size; @@ -816,12 +810,13 @@ static bool g4x_compute_srwm(struct drm_device *dev, return false; } - crtc = intel_get_crtc_for_plane(dev, plane); - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + crtc = intel_get_crtc_for_plane(dev_priv, plane); + adjusted_mode = &crtc->config->base.adjusted_mode; + fb = crtc->base.primary->state->fb; clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; - hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; - cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + hdisplay = crtc->config->pipe_src_w; + cpp = drm_format_plane_cpp(fb->pixel_format, 0); line_time_us = max(htotal * 1000 / clock, 1); line_count = (latency_ns / line_time_us + 1000) / 1000; @@ -835,11 +830,11 @@ static bool g4x_compute_srwm(struct drm_device *dev, *display_wm = entries + display->guard_size; /* calculate the self-refresh watermark for display cursor */ - entries = line_count * cpp * crtc->cursor->state->crtc_w; + entries = line_count * cpp * crtc->base.cursor->state->crtc_w; entries = DIV_ROUND_UP(entries, cursor->cacheline_size); *cursor_wm = entries + cursor->guard_size; - return g4x_check_srwm(dev, + return g4x_check_srwm(dev_priv, *display_wm, *cursor_wm, display, cursor); } @@ -939,10 +934,8 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate, return ret; } -static void vlv_setup_wm_latency(struct drm_device *dev) +static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* all latencies in usec */ dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; @@ -1067,7 +1060,8 @@ static void vlv_invert_wms(struct intel_crtc *crtc) for (level = 0; level < wm_state->num_levels; level++) { struct drm_device *dev = crtc->base.dev; - const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1; + const int sr_fifo_size = + INTEL_INFO(to_i915(dev))->num_pipes * 512 - 1; struct intel_plane *plane; wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane; @@ -1097,15 +1091,16 @@ static void vlv_invert_wms(struct intel_crtc *crtc) static void vlv_compute_wm(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); struct vlv_wm_state *wm_state = &crtc->wm_state; struct intel_plane *plane; - int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1; + int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1; int level; memset(wm_state, 0, sizeof(*wm_state)); wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed; - wm_state->num_levels = to_i915(dev)->wm.max_level + 1; + wm_state->num_levels = dev_priv->wm.max_level + 1; wm_state->num_active_planes = 0; @@ -1185,7 +1180,7 @@ static void vlv_compute_wm(struct intel_crtc *crtc) } /* clear any (partially) filled invalid levels */ - for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) { + for (level = wm_state->num_levels; level < dev_priv->wm.max_level + 1; level++) { memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level])); memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level])); } @@ -1329,20 +1324,19 @@ static void vlv_merge_wm(struct drm_device *dev, } } -static void vlv_update_wm(struct drm_crtc *crtc) +static void vlv_update_wm(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; + struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; + enum pipe pipe = crtc->pipe; struct vlv_wm_values wm = {}; - vlv_compute_wm(intel_crtc); + vlv_compute_wm(crtc); vlv_merge_wm(dev, &wm); if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) { /* FIXME should be part of crtc atomic commit */ - vlv_pipe_set_fifo_size(intel_crtc); + vlv_pipe_set_fifo_size(crtc); return; } @@ -1358,9 +1352,9 @@ static void vlv_update_wm(struct drm_crtc *crtc) intel_set_memory_cxsr(dev_priv, false); /* FIXME should be part of crtc atomic commit */ - vlv_pipe_set_fifo_size(intel_crtc); + vlv_pipe_set_fifo_size(crtc); - vlv_write_wm_values(intel_crtc, &wm); + vlv_write_wm_values(crtc, &wm); DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, " "sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n", @@ -1384,30 +1378,29 @@ static void vlv_update_wm(struct drm_crtc *crtc) #define single_plane_enabled(mask) is_power_of_2(mask) -static void g4x_update_wm(struct drm_crtc *crtc) +static void g4x_update_wm(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); static const int sr_latency_ns = 12000; - struct drm_i915_private *dev_priv = to_i915(dev); int planea_wm, planeb_wm, cursora_wm, cursorb_wm; int plane_sr, cursor_sr; unsigned int enabled = 0; bool cxsr_enabled; - if (g4x_compute_wm0(dev, PIPE_A, + if (g4x_compute_wm0(dev_priv, PIPE_A, &g4x_wm_info, pessimal_latency_ns, &g4x_cursor_wm_info, pessimal_latency_ns, &planea_wm, &cursora_wm)) enabled |= 1 << PIPE_A; - if (g4x_compute_wm0(dev, PIPE_B, + if (g4x_compute_wm0(dev_priv, PIPE_B, &g4x_wm_info, pessimal_latency_ns, &g4x_cursor_wm_info, pessimal_latency_ns, &planeb_wm, &cursorb_wm)) enabled |= 1 << PIPE_B; if (single_plane_enabled(enabled) && - g4x_compute_srwm(dev, ffs(enabled) - 1, + g4x_compute_srwm(dev_priv, ffs(enabled) - 1, sr_latency_ns, &g4x_wm_info, &g4x_cursor_wm_info, @@ -1442,25 +1435,27 @@ static void g4x_update_wm(struct drm_crtc *crtc) intel_set_memory_cxsr(dev_priv, true); } -static void i965_update_wm(struct drm_crtc *unused_crtc) +static void i965_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); + struct intel_crtc *crtc; int srwm = 1; int cursor_sr = 16; bool cxsr_enabled; /* Calc sr entries for one plane configs */ - crtc = single_enabled_crtc(dev); + crtc = single_enabled_crtc(dev_priv); if (crtc) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 12000; - const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; - int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + int hdisplay = crtc->config->pipe_src_w; + int cpp = drm_format_plane_cpp(fb->pixel_format, 0); unsigned long line_time_us; int entries; @@ -1478,7 +1473,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) entries, srwm); entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * - cpp * crtc->cursor->state->crtc_w; + cpp * crtc->base.cursor->state->crtc_w; entries = DIV_ROUND_UP(entries, i965_cursor_wm_info.cacheline_size); cursor_sr = i965_cursor_wm_info.fifo_size - @@ -1516,34 +1511,38 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) #undef FW_WM -static void i9xx_update_wm(struct drm_crtc *unused_crtc) +static void i9xx_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); const struct intel_watermark_params *wm_info; uint32_t fwater_lo; uint32_t fwater_hi; int cwm, srwm = 1; int fifo_size; int planea_wm, planeb_wm; - struct drm_crtc *crtc, *enabled = NULL; + struct intel_crtc *crtc, *enabled = NULL; - if (IS_I945GM(dev)) + if (IS_I945GM(dev_priv)) wm_info = &i945_wm_info; else if (!IS_GEN2(dev_priv)) wm_info = &i915_wm_info; else wm_info = &i830_a_wm_info; - fifo_size = dev_priv->display.get_fifo_size(dev, 0); - crtc = intel_get_crtc_for_plane(dev, 0); + fifo_size = dev_priv->display.get_fifo_size(dev_priv, 0); + crtc = intel_get_crtc_for_plane(dev_priv, 0); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; + int cpp; + if (IS_GEN2(dev_priv)) cpp = 4; + else + cpp = drm_format_plane_cpp(fb->pixel_format, 0); - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); @@ -1557,15 +1556,20 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) if (IS_GEN2(dev_priv)) wm_info = &i830_bc_wm_info; - fifo_size = dev_priv->display.get_fifo_size(dev, 1); - crtc = intel_get_crtc_for_plane(dev, 1); + fifo_size = dev_priv->display.get_fifo_size(dev_priv, 1); + crtc = intel_get_crtc_for_plane(dev_priv, 1); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; + int cpp; + if (IS_GEN2(dev_priv)) cpp = 4; + else + cpp = drm_format_plane_cpp(fb->pixel_format, 0); - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); @@ -1584,7 +1588,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) if (IS_I915GM(dev_priv) && enabled) { struct drm_i915_gem_object *obj; - obj = intel_fb_obj(enabled->primary->state->fb); + obj = intel_fb_obj(enabled->base.primary->state->fb); /* self-refresh seems busted with untiled */ if (!i915_gem_object_is_tiled(obj)) @@ -1600,19 +1604,24 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) intel_set_memory_cxsr(dev_priv, false); /* Calc sr entries for one plane configs */ - if (HAS_FW_BLC(dev) && enabled) { + if (HAS_FW_BLC(dev_priv) && enabled) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 6000; - const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = + &enabled->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + enabled->base.primary->state->fb; int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; - int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w; - int cpp = drm_format_plane_cpp(enabled->primary->state->fb->pixel_format, 0); + int hdisplay = enabled->config->pipe_src_w; + int cpp; unsigned long line_time_us; int entries; if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) cpp = 4; + else + cpp = drm_format_plane_cpp(fb->pixel_format, 0); line_time_us = max(htotal * 1000 / clock, 1); @@ -1649,23 +1658,22 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) intel_set_memory_cxsr(dev_priv, true); } -static void i845_update_wm(struct drm_crtc *unused_crtc) +static void i845_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); + struct intel_crtc *crtc; const struct drm_display_mode *adjusted_mode; uint32_t fwater_lo; int planea_wm; - crtc = single_enabled_crtc(dev); + crtc = single_enabled_crtc(dev_priv); if (crtc == NULL) return; - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + adjusted_mode = &crtc->config->base.adjusted_mode; planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, &i845_wm_info, - dev_priv->display.get_fifo_size(dev, 0), + dev_priv->display.get_fifo_size(dev_priv, 0), 4, pessimal_latency_ns); fwater_lo = I915_READ(FW_BLC) & ~0xfff; fwater_lo |= (3<<8) | planea_wm; @@ -1854,23 +1862,25 @@ static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate, return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp); } -static unsigned int ilk_display_fifo_size(const struct drm_device *dev) +static unsigned int +ilk_display_fifo_size(const struct drm_i915_private *dev_priv) { - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_GEN(dev_priv) >= 8) return 3072; - else if (INTEL_INFO(dev)->gen >= 7) + else if (INTEL_GEN(dev_priv) >= 7) return 768; else return 512; } -static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev, - int level, bool is_sprite) +static unsigned int +ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv, + int level, bool is_sprite) { - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_GEN(dev_priv) >= 8) /* BDW primary/sprite plane watermarks */ return level == 0 ? 255 : 2047; - else if (INTEL_INFO(dev)->gen >= 7) + else if (INTEL_GEN(dev_priv) >= 7) /* IVB/HSW primary/sprite plane watermarks */ return level == 0 ? 127 : 1023; else if (!is_sprite) @@ -1881,18 +1891,18 @@ static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev, return level == 0 ? 63 : 255; } -static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev, - int level) +static unsigned int +ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level) { - if (INTEL_INFO(dev)->gen >= 7) + if (INTEL_GEN(dev_priv) >= 7) return level == 0 ? 63 : 255; else return level == 0 ? 31 : 63; } -static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev) +static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv) { - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_GEN(dev_priv) >= 8) return 31; else return 15; @@ -1905,7 +1915,8 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev, enum intel_ddb_partitioning ddb_partitioning, bool is_sprite) { - unsigned int fifo_size = ilk_display_fifo_size(dev); + struct drm_i915_private *dev_priv = to_i915(dev); + unsigned int fifo_size = ilk_display_fifo_size(dev_priv); /* if sprites aren't enabled, sprites get nothing */ if (is_sprite && !config->sprites_enabled) @@ -1913,14 +1924,14 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev, /* HSW allows LP1+ watermarks even with multiple pipes */ if (level == 0 || config->num_pipes_active > 1) { - fifo_size /= INTEL_INFO(dev)->num_pipes; + fifo_size /= INTEL_INFO(dev_priv)->num_pipes; /* * For some reason the non self refresh * FIFO size is only half of the self * refresh FIFO size on ILK/SNB. */ - if (INTEL_INFO(dev)->gen <= 6) + if (INTEL_GEN(dev_priv) <= 6) fifo_size /= 2; } @@ -1936,7 +1947,7 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev, } /* clamp to max that the registers can hold */ - return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite)); + return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite)); } /* Calculate the maximum cursor plane watermark */ @@ -1949,7 +1960,7 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, return 64; /* otherwise just report max that registers can hold */ - return ilk_cursor_wm_reg_max(dev, level); + return ilk_cursor_wm_reg_max(to_i915(dev), level); } static void ilk_compute_wm_maximums(const struct drm_device *dev, @@ -1961,17 +1972,17 @@ static void ilk_compute_wm_maximums(const struct drm_device *dev, max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false); max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true); max->cur = ilk_cursor_wm_max(dev, level, config); - max->fbc = ilk_fbc_wm_reg_max(dev); + max->fbc = ilk_fbc_wm_reg_max(to_i915(dev)); } -static void ilk_compute_wm_reg_maximums(struct drm_device *dev, +static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv, int level, struct ilk_wm_maximums *max) { - max->pri = ilk_plane_wm_reg_max(dev, level, false); - max->spr = ilk_plane_wm_reg_max(dev, level, true); - max->cur = ilk_cursor_wm_reg_max(dev, level); - max->fbc = ilk_fbc_wm_reg_max(dev); + max->pri = ilk_plane_wm_reg_max(dev_priv, level, false); + max->spr = ilk_plane_wm_reg_max(dev_priv, level, true); + max->cur = ilk_cursor_wm_reg_max(dev_priv, level); + max->fbc = ilk_fbc_wm_reg_max(dev_priv); } static bool ilk_validate_wm_level(int level, @@ -2078,10 +2089,9 @@ hsw_compute_linetime_wm(const struct intel_crtc_state *cstate) PIPE_WM_LINETIME_TIME(linetime); } -static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) +static void intel_read_wm_latency(struct drm_i915_private *dev_priv, + uint16_t wm[8]) { - struct drm_i915_private *dev_priv = to_i915(dev); - if (IS_GEN9(dev_priv)) { uint32_t val; int ret, i; @@ -2167,14 +2177,14 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) wm[2] = (sskpd >> 12) & 0xFF; wm[3] = (sskpd >> 20) & 0x1FF; wm[4] = (sskpd >> 32) & 0x1FF; - } else if (INTEL_INFO(dev)->gen >= 6) { + } else if (INTEL_GEN(dev_priv) >= 6) { uint32_t sskpd = I915_READ(MCH_SSKPD); wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK; wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK; wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK; wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK; - } else if (INTEL_INFO(dev)->gen >= 5) { + } else if (INTEL_GEN(dev_priv) >= 5) { uint32_t mltr = I915_READ(MLTR_ILK); /* ILK primary LP0 latency is 700 ns */ @@ -2262,9 +2272,8 @@ static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, return true; } -static void snb_wm_latency_quirk(struct drm_device *dev) +static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); bool changed; /* @@ -2284,11 +2293,9 @@ static void snb_wm_latency_quirk(struct drm_device *dev) intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); } -static void ilk_setup_wm_latency(struct drm_device *dev) +static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - intel_read_wm_latency(dev, dev_priv->wm.pri_latency); + intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency); memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, sizeof(dev_priv->wm.pri_latency)); @@ -2303,14 +2310,12 @@ static void ilk_setup_wm_latency(struct drm_device *dev) intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); if (IS_GEN6(dev_priv)) - snb_wm_latency_quirk(dev); + snb_wm_latency_quirk(dev_priv); } -static void skl_setup_wm_latency(struct drm_device *dev) +static void skl_setup_wm_latency(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - intel_read_wm_latency(dev, dev_priv->wm.skl_latency); + intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency); intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency); } @@ -2381,7 +2386,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) usable_level = max_level; /* ILK/SNB: LP2+ watermarks only w/o sprites */ - if (INTEL_INFO(dev)->gen <= 6 && pipe_wm->sprites_enabled) + if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled) usable_level = 1; /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ @@ -2400,7 +2405,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) if (!ilk_validate_pipe_wm(dev, pipe_wm)) return -EINVAL; - ilk_compute_wm_reg_maximums(dev, 1, &max); + ilk_compute_wm_reg_maximums(dev_priv, 1, &max); for (level = 1; level <= max_level; level++) { struct intel_wm_level *wm = &pipe_wm->raw_wm[level]; @@ -2529,7 +2534,7 @@ static void ilk_wm_merge(struct drm_device *dev, last_enabled_level = 0; /* ILK: FBC WM must be disabled always */ - merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6; + merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6; /* merge each WM1+ level */ for (level = 1; level <= max_level; level++) { @@ -2592,6 +2597,7 @@ static void ilk_compute_wm_results(struct drm_device *dev, enum intel_ddb_partitioning partitioning, struct ilk_wm_values *results) { + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc; int level, wm_lp; @@ -2618,7 +2624,7 @@ static void ilk_compute_wm_results(struct drm_device *dev, if (r->enable) results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN; - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_GEN(dev_priv) >= 8) results->wm_lp[wm_lp - 1] |= r->fbc_val << WM1_LP_FBC_SHIFT_BDW; else @@ -2629,7 +2635,7 @@ static void ilk_compute_wm_results(struct drm_device *dev, * Always set WM1S_LP_EN when spr_val != 0, even if the * level is disabled. Doing otherwise could cause underruns. */ - if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) { + if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) { WARN_ON(wm_lp != 1); results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; } else @@ -2779,7 +2785,6 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, static void ilk_write_wm_values(struct drm_i915_private *dev_priv, struct ilk_wm_values *results) { - struct drm_device *dev = &dev_priv->drm; struct ilk_wm_values *previous = &dev_priv->wm.hw; unsigned int dirty; uint32_t val; @@ -2835,7 +2840,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, previous->wm_lp_spr[0] != results->wm_lp_spr[0]) I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]); - if (INTEL_INFO(dev)->gen >= 7) { + if (INTEL_GEN(dev_priv) >= 7) { if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) @@ -3041,7 +3046,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) /* Since we're now guaranteed to only have one active CRTC... */ pipe = ffs(intel_state->active_crtcs) - 1; - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); cstate = to_intel_crtc_state(crtc->base.state); if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) @@ -3062,7 +3067,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) latency = dev_priv->wm.skl_latency[level]; if (skl_needs_memory_bw_wa(intel_state) && - plane->base.state->fb->modifier[0] == + plane->base.state->fb->modifier == I915_FORMAT_MOD_X_TILED) latency += 15; @@ -3117,7 +3122,11 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, * we currently hold. */ if (!intel_state->active_pipe_changes) { - *alloc = to_intel_crtc(for_crtc)->hw_ddb; + /* + * alloc may be cleared by clear_intel_crtc_state, + * copy from old state to be sure + */ + *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb; return; } @@ -3160,7 +3169,7 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) continue; - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { val = I915_READ(PLANE_BUF_CFG(pipe, plane)); skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane], val); @@ -3262,49 +3271,39 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, * 3 * 4096 * 8192 * 4 < 2^32 */ static unsigned int -skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate) +skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate, + unsigned *plane_data_rate, + unsigned *plane_y_data_rate) { struct drm_crtc_state *cstate = &intel_cstate->base; struct drm_atomic_state *state = cstate->state; - struct drm_crtc *crtc = cstate->crtc; - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - const struct drm_plane *plane; + struct drm_plane *plane; const struct intel_plane *intel_plane; - struct drm_plane_state *pstate; + const struct drm_plane_state *pstate; unsigned int rate, total_data_rate = 0; int id; - int i; if (WARN_ON(!state)) return 0; /* Calculate and cache data rate for each plane */ - for_each_plane_in_state(state, plane, pstate, i) { + drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) { id = skl_wm_plane_id(to_intel_plane(plane)); intel_plane = to_intel_plane(plane); - if (intel_plane->pipe != intel_crtc->pipe) - continue; - /* packed/uv */ rate = skl_plane_relative_data_rate(intel_cstate, pstate, 0); - intel_cstate->wm.skl.plane_data_rate[id] = rate; + plane_data_rate[id] = rate; + + total_data_rate += rate; /* y-plane */ rate = skl_plane_relative_data_rate(intel_cstate, pstate, 1); - intel_cstate->wm.skl.plane_y_data_rate[id] = rate; - } + plane_y_data_rate[id] = rate; - /* Calculate CRTC's total data rate from cached values */ - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { - int id = skl_wm_plane_id(intel_plane); - - /* packed/uv */ - total_data_rate += intel_cstate->wm.skl.plane_data_rate[id]; - total_data_rate += intel_cstate->wm.skl.plane_y_data_rate[id]; + total_data_rate += rate; } return total_data_rate; @@ -3328,8 +3327,8 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate, return 0; /* For Non Y-tile return 8-blocks */ - if (fb->modifier[0] != I915_FORMAT_MOD_Y_TILED && - fb->modifier[0] != I915_FORMAT_MOD_Yf_TILED) + if (fb->modifier != I915_FORMAT_MOD_Y_TILED && + fb->modifier != I915_FORMAT_MOD_Yf_TILED) return 8; src_w = drm_rect_width(&intel_pstate->base.src) >> 16; @@ -3373,6 +3372,30 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate, return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3; } +static void +skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active, + uint16_t *minimum, uint16_t *y_minimum) +{ + const struct drm_plane_state *pstate; + struct drm_plane *plane; + + drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) { + struct intel_plane *intel_plane = to_intel_plane(plane); + int id = skl_wm_plane_id(intel_plane); + + if (id == PLANE_CURSOR) + continue; + + if (!pstate->visible) + continue; + + minimum[id] = skl_ddb_min_alloc(pstate, 0); + y_minimum[id] = skl_ddb_min_alloc(pstate, 1); + } + + minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active); +} + static int skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, struct skl_ddb_allocation *ddb /* out */) @@ -3381,17 +3404,16 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, struct drm_crtc *crtc = cstate->base.crtc; struct drm_device *dev = crtc->dev; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_plane *intel_plane; - struct drm_plane *plane; - struct drm_plane_state *pstate; enum pipe pipe = intel_crtc->pipe; struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb; - uint16_t alloc_size, start, cursor_blocks; - uint16_t *minimum = cstate->wm.skl.minimum_blocks; - uint16_t *y_minimum = cstate->wm.skl.minimum_y_blocks; + uint16_t alloc_size, start; + uint16_t minimum[I915_MAX_PLANES] = {}; + uint16_t y_minimum[I915_MAX_PLANES] = {}; unsigned int total_data_rate; int num_active; int id, i; + unsigned plane_data_rate[I915_MAX_PLANES] = {}; + unsigned plane_y_data_rate[I915_MAX_PLANES] = {}; /* Clear the partitioning for disabled planes. */ memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); @@ -3412,57 +3434,43 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, return 0; } - cursor_blocks = skl_cursor_allocation(num_active); - ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks; - ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; - - alloc_size -= cursor_blocks; - - /* 1. Allocate the mininum required blocks for each active plane */ - for_each_plane_in_state(state, plane, pstate, i) { - intel_plane = to_intel_plane(plane); - id = skl_wm_plane_id(intel_plane); - - if (intel_plane->pipe != pipe) - continue; + skl_ddb_calc_min(cstate, num_active, minimum, y_minimum); - if (!to_intel_plane_state(pstate)->base.visible) { - minimum[id] = 0; - y_minimum[id] = 0; - continue; - } - if (plane->type == DRM_PLANE_TYPE_CURSOR) { - minimum[id] = 0; - y_minimum[id] = 0; - continue; - } - - minimum[id] = skl_ddb_min_alloc(pstate, 0); - y_minimum[id] = skl_ddb_min_alloc(pstate, 1); - } + /* + * 1. Allocate the mininum required blocks for each active plane + * and allocate the cursor, it doesn't require extra allocation + * proportional to the data rate. + */ - for (i = 0; i < PLANE_CURSOR; i++) { + for (i = 0; i < I915_MAX_PLANES; i++) { alloc_size -= minimum[i]; alloc_size -= y_minimum[i]; } + ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR]; + ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; + /* * 2. Distribute the remaining space in proportion to the amount of * data each plane needs to fetch from memory. * * FIXME: we may not allocate every single block here. */ - total_data_rate = skl_get_total_relative_data_rate(cstate); + total_data_rate = skl_get_total_relative_data_rate(cstate, + plane_data_rate, + plane_y_data_rate); if (total_data_rate == 0) return 0; start = alloc->start; - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + for (id = 0; id < I915_MAX_PLANES; id++) { unsigned int data_rate, y_data_rate; uint16_t plane_blocks, y_plane_blocks = 0; - int id = skl_wm_plane_id(intel_plane); - data_rate = cstate->wm.skl.plane_data_rate[id]; + if (id == PLANE_CURSOR) + continue; + + data_rate = plane_data_rate[id]; /* * allocation for (packed formats) or (uv-plane part of planar format): @@ -3484,7 +3492,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, /* * allocation for y_plane part of planar format: */ - y_data_rate = cstate->wm.skl.plane_y_data_rate[id]; + y_data_rate = plane_y_data_rate[id]; y_plane_blocks = y_minimum[id]; y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate, @@ -3589,7 +3597,7 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, return 0; } - if (apply_memory_bw_wa && fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (apply_memory_bw_wa && fb->modifier == I915_FORMAT_MOD_X_TILED) latency += 15; width = drm_rect_width(&intel_pstate->base.src) >> 16; @@ -3624,13 +3632,16 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, y_min_scanlines = 4; } + if (apply_memory_bw_wa) + y_min_scanlines *= 2; + plane_bytes_per_line = width * cpp; - if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || - fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { + if (fb->modifier == I915_FORMAT_MOD_Y_TILED || + fb->modifier == I915_FORMAT_MOD_Yf_TILED) { plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line * y_min_scanlines, 512); plane_blocks_per_line /= y_min_scanlines; - } else if (fb->modifier[0] == DRM_FORMAT_MOD_NONE) { + } else if (fb->modifier == DRM_FORMAT_MOD_NONE) { plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512) + 1; } else { @@ -3644,11 +3655,9 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, plane_blocks_per_line); y_tile_minimum = plane_blocks_per_line * y_min_scanlines; - if (apply_memory_bw_wa) - y_tile_minimum *= 2; - if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || - fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { + if (fb->modifier == I915_FORMAT_MOD_Y_TILED || + fb->modifier == I915_FORMAT_MOD_Yf_TILED) { selected_result = max(method2, y_tile_minimum); } else { if ((cpp * cstate->base.adjusted_mode.crtc_htotal / 512 < 1) && @@ -3664,8 +3673,8 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line); if (level >= 1 && level <= 7) { - if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || - fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { + if (fb->modifier == I915_FORMAT_MOD_Y_TILED || + fb->modifier == I915_FORMAT_MOD_Yf_TILED) { res_blocks += y_tile_minimum; res_lines += y_min_scanlines; } else { @@ -3842,10 +3851,10 @@ static void skl_write_wm_level(struct drm_i915_private *dev_priv, I915_WRITE(reg, val); } -void skl_write_plane_wm(struct intel_crtc *intel_crtc, - const struct skl_plane_wm *wm, - const struct skl_ddb_allocation *ddb, - int plane) +static void skl_write_plane_wm(struct intel_crtc *intel_crtc, + const struct skl_plane_wm *wm, + const struct skl_ddb_allocation *ddb, + int plane) { struct drm_crtc *crtc = &intel_crtc->base; struct drm_device *dev = crtc->dev; @@ -3866,9 +3875,9 @@ void skl_write_plane_wm(struct intel_crtc *intel_crtc, &ddb->y_plane[pipe][plane]); } -void skl_write_cursor_wm(struct intel_crtc *intel_crtc, - const struct skl_plane_wm *wm, - const struct skl_ddb_allocation *ddb) +static void skl_write_cursor_wm(struct intel_crtc *intel_crtc, + const struct skl_plane_wm *wm, + const struct skl_ddb_allocation *ddb) { struct drm_crtc *crtc = &intel_crtc->base; struct drm_device *dev = crtc->dev; @@ -3906,35 +3915,26 @@ static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, return a->start < b->end && b->start < a->end; } -bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state, - struct intel_crtc *intel_crtc) +bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries, + const struct skl_ddb_entry *ddb, + int ignore) { - struct drm_crtc *other_crtc; - struct drm_crtc_state *other_cstate; - struct intel_crtc *other_intel_crtc; - const struct skl_ddb_entry *ddb = - &to_intel_crtc_state(intel_crtc->base.state)->wm.skl.ddb; int i; - for_each_crtc_in_state(state, other_crtc, other_cstate, i) { - other_intel_crtc = to_intel_crtc(other_crtc); - - if (other_intel_crtc == intel_crtc) - continue; - - if (skl_ddb_entries_overlap(ddb, &other_intel_crtc->hw_ddb)) + for (i = 0; i < I915_MAX_PIPES; i++) + if (i != ignore && entries[i] && + skl_ddb_entries_overlap(ddb, entries[i])) return true; - } return false; } static int skl_update_pipe_wm(struct drm_crtc_state *cstate, - struct skl_ddb_allocation *ddb, /* out */ + const struct skl_pipe_wm *old_pipe_wm, struct skl_pipe_wm *pipe_wm, /* out */ + struct skl_ddb_allocation *ddb, /* out */ bool *changed /* out */) { - struct intel_crtc *intel_crtc = to_intel_crtc(cstate->crtc); struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate); int ret; @@ -3942,7 +3942,7 @@ static int skl_update_pipe_wm(struct drm_crtc_state *cstate, if (ret) return ret; - if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm))) + if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm))) *changed = false; else *changed = true; @@ -3981,7 +3981,7 @@ skl_ddb_add_affected_planes(struct intel_crtc_state *cstate) WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc)); - drm_for_each_plane_mask(plane, dev, crtc->state->plane_mask) { + drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) { id = skl_wm_plane_id(to_intel_plane(plane)); if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][id], @@ -4096,45 +4096,31 @@ skl_print_wm_changes(const struct drm_atomic_state *state) to_intel_atomic_state(state); const struct drm_crtc *crtc; const struct drm_crtc_state *cstate; - const struct drm_plane *plane; const struct intel_plane *intel_plane; - const struct drm_plane_state *pstate; const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb; const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb; - enum pipe pipe; int id; - int i, j; + int i; for_each_crtc_in_state(state, crtc, cstate, i) { - pipe = to_intel_crtc(crtc)->pipe; + const struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + enum pipe pipe = intel_crtc->pipe; - for_each_plane_in_state(state, plane, pstate, j) { + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { const struct skl_ddb_entry *old, *new; - intel_plane = to_intel_plane(plane); id = skl_wm_plane_id(intel_plane); old = &old_ddb->plane[pipe][id]; new = &new_ddb->plane[pipe][id]; - if (intel_plane->pipe != pipe) - continue; - if (skl_ddb_entry_equal(old, new)) continue; - if (id != PLANE_CURSOR) { - DRM_DEBUG_ATOMIC("[PLANE:%d:plane %d%c] ddb (%d - %d) -> (%d - %d)\n", - plane->base.id, id + 1, - pipe_name(pipe), - old->start, old->end, - new->start, new->end); - } else { - DRM_DEBUG_ATOMIC("[PLANE:%d:cursor %c] ddb (%d - %d) -> (%d - %d)\n", - plane->base.id, - pipe_name(pipe), - old->start, old->end, - new->start, new->end); - } + DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n", + intel_plane->base.base.id, + intel_plane->base.name, + old->start, old->end, + new->start, new->end); } } } @@ -4183,10 +4169,12 @@ skl_compute_wm(struct drm_atomic_state *state) for_each_crtc_in_state(state, crtc, cstate, i) { struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate); + const struct skl_pipe_wm *old_pipe_wm = + &to_intel_crtc_state(crtc->state)->wm.skl.optimal; pipe_wm = &intel_cstate->wm.skl.optimal; - ret = skl_update_pipe_wm(cstate, &results->ddb, pipe_wm, - &changed); + ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm, + &results->ddb, &changed); if (ret) return ret; @@ -4205,44 +4193,46 @@ skl_compute_wm(struct drm_atomic_state *state) return 0; } -static void skl_update_wm(struct drm_crtc *crtc) +static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state, + struct intel_crtc_state *cstate) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct skl_wm_values *results = &dev_priv->wm.skl_results; - struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw; - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); + struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc); + struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal; - enum pipe pipe = intel_crtc->pipe; + const struct skl_ddb_allocation *ddb = &state->wm_results.ddb; + enum pipe pipe = crtc->pipe; + int plane; - if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0) + if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base))) return; - intel_crtc->wm.active.skl = *pipe_wm; + I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime); - mutex_lock(&dev_priv->wm.wm_mutex); + for_each_universal_plane(dev_priv, pipe, plane) + skl_write_plane_wm(crtc, &pipe_wm->planes[plane], ddb, plane); - /* - * If this pipe isn't active already, we're going to be enabling it - * very soon. Since it's safe to update a pipe's ddb allocation while - * the pipe's shut off, just do so here. Already active pipes will have - * their watermarks updated once we update their planes. - */ - if (crtc->state->active_changed) { - int plane; + skl_write_cursor_wm(crtc, &pipe_wm->planes[PLANE_CURSOR], ddb); +} + +static void skl_initial_wm(struct intel_atomic_state *state, + struct intel_crtc_state *cstate) +{ + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); + struct drm_device *dev = intel_crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct skl_wm_values *results = &state->wm_results; + struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw; + enum pipe pipe = intel_crtc->pipe; - for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) - skl_write_plane_wm(intel_crtc, &pipe_wm->planes[plane], - &results->ddb, plane); + if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0) + return; - skl_write_cursor_wm(intel_crtc, &pipe_wm->planes[PLANE_CURSOR], - &results->ddb); - } + mutex_lock(&dev_priv->wm.wm_mutex); - skl_copy_wm_for_pipe(hw_vals, results, pipe); + if (cstate->base.active_changed) + skl_atomic_update_crtc_wm(state, cstate); - intel_crtc->hw_ddb = cstate->wm.skl.ddb; + skl_copy_wm_for_pipe(hw_vals, results, pipe); mutex_unlock(&dev_priv->wm.wm_mutex); } @@ -4280,7 +4270,7 @@ static void ilk_program_watermarks(struct drm_i915_private *dev_priv) ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); /* 5/6 split only in single pipe config on IVB+ */ - if (INTEL_INFO(dev)->gen >= 7 && + if (INTEL_GEN(dev_priv) >= 7 && config.num_pipes_active == 1 && config.sprites_enabled) { ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max); ilk_wm_merge(dev, &config, &max, &lp_wm_5_6); @@ -4298,7 +4288,8 @@ static void ilk_program_watermarks(struct drm_i915_private *dev_priv) ilk_write_wm_values(dev_priv, &results); } -static void ilk_initial_watermarks(struct intel_crtc_state *cstate) +static void ilk_initial_watermarks(struct intel_atomic_state *state, + struct intel_crtc_state *cstate) { struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); @@ -4309,7 +4300,8 @@ static void ilk_initial_watermarks(struct intel_crtc_state *cstate) mutex_unlock(&dev_priv->wm.wm_mutex); } -static void ilk_optimize_watermarks(struct intel_crtc_state *cstate) +static void ilk_optimize_watermarks(struct intel_atomic_state *state, + struct intel_crtc_state *cstate) { struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); @@ -4388,10 +4380,8 @@ void skl_wm_get_hw_state(struct drm_device *dev) skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal); - if (intel_crtc->active) { + if (intel_crtc->active) hw->dirty_pipes |= drm_crtc_mask(crtc); - intel_crtc->wm.active.skl = cstate->wm.skl.optimal; - } } if (dev_priv->active_crtcs) { @@ -4553,11 +4543,11 @@ void vlv_wm_get_hw_state(struct drm_device *dev) plane->wm.fifo_size = 63; break; case DRM_PLANE_TYPE_PRIMARY: - plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0); + plane->wm.fifo_size = vlv_get_fifo_size(dev_priv, plane->pipe, 0); break; case DRM_PLANE_TYPE_OVERLAY: sprite = plane->plane; - plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1); + plane->wm.fifo_size = vlv_get_fifo_size(dev_priv, plane->pipe, sprite + 1); break; } } @@ -4622,7 +4612,7 @@ void ilk_wm_get_hw_state(struct drm_device *dev) hw->wm_lp[2] = I915_READ(WM3_LP_ILK); hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK); - if (INTEL_INFO(dev)->gen >= 7) { + if (INTEL_GEN(dev_priv) >= 7) { hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); } @@ -4670,9 +4660,9 @@ void ilk_wm_get_hw_state(struct drm_device *dev) * We don't use the sprite, so we can ignore that. And on Crestline we have * to set the non-SR watermarks to 8. */ -void intel_update_watermarks(struct drm_crtc *crtc) +void intel_update_watermarks(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); if (dev_priv->display.update_wm) dev_priv->display.update_wm(crtc); @@ -5878,7 +5868,7 @@ static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv) if (WARN_ON(!dev_priv->vlv_pctx)) return; - i915_gem_object_put_unlocked(dev_priv->vlv_pctx); + i915_gem_object_put(dev_priv->vlv_pctx); dev_priv->vlv_pctx = NULL; } @@ -6862,10 +6852,8 @@ void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv) } } -static void ibx_init_clock_gating(struct drm_device *dev) +static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* * On Ibex Peak and Cougar Point, we need to disable clock * gating for the panel power sequencer or it will fail to @@ -6874,9 +6862,8 @@ static void ibx_init_clock_gating(struct drm_device *dev) I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); } -static void g4x_disable_trickle_feed(struct drm_device *dev) +static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; for_each_pipe(dev_priv, pipe) { @@ -6889,10 +6876,8 @@ static void g4x_disable_trickle_feed(struct drm_device *dev) } } -static void ilk_init_lp_watermarks(struct drm_device *dev) +static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN); @@ -6903,9 +6888,8 @@ static void ilk_init_lp_watermarks(struct drm_device *dev) */ } -static void ironlake_init_clock_gating(struct drm_device *dev) +static void ironlake_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; /* @@ -6937,7 +6921,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev) (I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS)); - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); /* * Based on the document from hardware guys the following bits @@ -6972,14 +6956,13 @@ static void ironlake_init_clock_gating(struct drm_device *dev) /* WaDisable_RenderCache_OperationalFlush:ilk */ I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); - ibx_init_clock_gating(dev); + ibx_init_clock_gating(dev_priv); } -static void cpt_init_clock_gating(struct drm_device *dev) +static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); int pipe; uint32_t val; @@ -7014,9 +6997,8 @@ static void cpt_init_clock_gating(struct drm_device *dev) } } -static void gen6_check_mch_setup(struct drm_device *dev) +static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t tmp; tmp = I915_READ(MCH_SSKPD); @@ -7025,9 +7007,8 @@ static void gen6_check_mch_setup(struct drm_device *dev) tmp); } -static void gen6_init_clock_gating(struct drm_device *dev) +static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); @@ -7054,7 +7035,7 @@ static void gen6_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN6_GT_MODE, _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); @@ -7115,11 +7096,11 @@ static void gen6_init_clock_gating(struct drm_device *dev) ILK_DPARBUNIT_CLOCK_GATE_ENABLE | ILK_DPFDUNIT_CLOCK_GATE_ENABLE); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); - cpt_init_clock_gating(dev); + cpt_init_clock_gating(dev_priv); - gen6_check_mch_setup(dev); + gen6_check_mch_setup(dev_priv); } static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) @@ -7140,10 +7121,8 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) I915_WRITE(GEN7_FF_THREAD_MODE, reg); } -static void lpt_init_clock_gating(struct drm_device *dev) +static void lpt_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* * TODO: this bit should only be enabled when really needed, then * disabled when not needed anymore in order to save power. @@ -7159,10 +7138,8 @@ static void lpt_init_clock_gating(struct drm_device *dev) TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); } -static void lpt_suspend_hw(struct drm_device *dev) +static void lpt_suspend_hw(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - if (HAS_PCH_LPT_LP(dev_priv)) { uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); @@ -7194,11 +7171,9 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv, I915_WRITE(GEN7_MISCCPCTL, misccpctl); } -static void kabylake_init_clock_gating(struct drm_device *dev) +static void kabylake_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - gen9_init_clock_gating(dev); + gen9_init_clock_gating(dev_priv); /* WaDisableSDEUnitClockGating:kbl */ if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) @@ -7215,11 +7190,9 @@ static void kabylake_init_clock_gating(struct drm_device *dev) ILK_DPFC_NUKE_ON_ANY_MODIFICATION); } -static void skylake_init_clock_gating(struct drm_device *dev) +static void skylake_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - gen9_init_clock_gating(dev); + gen9_init_clock_gating(dev_priv); /* WAC6entrylatency:skl */ I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) | @@ -7230,12 +7203,11 @@ static void skylake_init_clock_gating(struct drm_device *dev) ILK_DPFC_NUKE_ON_ANY_MODIFICATION); } -static void broadwell_init_clock_gating(struct drm_device *dev) +static void broadwell_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); /* WaSwitchSolVfFArbitrationPriority:bdw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -7278,14 +7250,12 @@ static void broadwell_init_clock_gating(struct drm_device *dev) I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1) | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT); - lpt_init_clock_gating(dev); + lpt_init_clock_gating(dev_priv); } -static void haswell_init_clock_gating(struct drm_device *dev) +static void haswell_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); /* L3 caching of data atomics doesn't work -- disable it. */ I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); @@ -7334,15 +7304,14 @@ static void haswell_init_clock_gating(struct drm_device *dev) I915_WRITE(CHICKEN_PAR1_1, I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES); - lpt_init_clock_gating(dev); + lpt_init_clock_gating(dev_priv); } -static void ivybridge_init_clock_gating(struct drm_device *dev) +static void ivybridge_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t snpcr; - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); @@ -7399,7 +7368,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); gen7_setup_fixed_func_scheduler(dev_priv); @@ -7430,15 +7399,13 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); if (!HAS_PCH_NOP(dev_priv)) - cpt_init_clock_gating(dev); + cpt_init_clock_gating(dev_priv); - gen6_check_mch_setup(dev); + gen6_check_mch_setup(dev_priv); } -static void valleyview_init_clock_gating(struct drm_device *dev) +static void valleyview_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* WaDisableEarlyCull:vlv */ I915_WRITE(_3D_CHICKEN3, _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); @@ -7517,10 +7484,8 @@ static void valleyview_init_clock_gating(struct drm_device *dev) I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); } -static void cherryview_init_clock_gating(struct drm_device *dev) +static void cherryview_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* WaVSRefCountFullforceMissDisable:chv */ /* WaDSRefCountFullforceMissDisable:chv */ I915_WRITE(GEN7_FF_THREAD_MODE, @@ -7553,9 +7518,8 @@ static void cherryview_init_clock_gating(struct drm_device *dev) I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL); } -static void g4x_init_clock_gating(struct drm_device *dev) +static void g4x_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate; I915_WRITE(RENCLK_GATE_D1, 0); @@ -7577,13 +7541,11 @@ static void g4x_init_clock_gating(struct drm_device *dev) /* WaDisable_RenderCache_OperationalFlush:g4x */ I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); } -static void crestline_init_clock_gating(struct drm_device *dev) +static void crestline_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); I915_WRITE(RENCLK_GATE_D2, 0); I915_WRITE(DSPCLK_GATE_D, 0); @@ -7596,10 +7558,8 @@ static void crestline_init_clock_gating(struct drm_device *dev) I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } -static void broadwater_init_clock_gating(struct drm_device *dev) +static void broadwater_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | I965_RCC_CLOCK_GATE_DISABLE | I965_RCPB_CLOCK_GATE_DISABLE | @@ -7613,16 +7573,15 @@ static void broadwater_init_clock_gating(struct drm_device *dev) I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } -static void gen3_init_clock_gating(struct drm_device *dev) +static void gen3_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 dstate = I915_READ(D_STATE); dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | DSTATE_DOT_CLOCK_GATING; I915_WRITE(D_STATE, dstate); - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); /* IIR "flip pending" means done if this bit is set */ @@ -7638,10 +7597,8 @@ static void gen3_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); } -static void i85x_init_clock_gating(struct drm_device *dev) +static void i85x_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); /* interrupts should cause a wake up from C3 */ @@ -7652,10 +7609,8 @@ static void i85x_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); } -static void i830_init_clock_gating(struct drm_device *dev) +static void i830_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); I915_WRITE(MEM_MODE, @@ -7663,20 +7618,18 @@ static void i830_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); } -void intel_init_clock_gating(struct drm_device *dev) +void intel_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - dev_priv->display.init_clock_gating(dev); + dev_priv->display.init_clock_gating(dev_priv); } -void intel_suspend_hw(struct drm_device *dev) +void intel_suspend_hw(struct drm_i915_private *dev_priv) { - if (HAS_PCH_LPT(to_i915(dev))) - lpt_suspend_hw(dev); + if (HAS_PCH_LPT(dev_priv)) + lpt_suspend_hw(dev_priv); } -static void nop_init_clock_gating(struct drm_device *dev) +static void nop_init_clock_gating(struct drm_i915_private *dev_priv) { DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n"); } @@ -7731,25 +7684,24 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) } /* Set up chip specific power management-related functions */ -void intel_init_pm(struct drm_device *dev) +void intel_init_pm(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - intel_fbc_init(dev_priv); /* For cxsr */ - if (IS_PINEVIEW(dev)) - i915_pineview_get_mem_freq(dev); + if (IS_PINEVIEW(dev_priv)) + i915_pineview_get_mem_freq(dev_priv); else if (IS_GEN5(dev_priv)) - i915_ironlake_get_mem_freq(dev); + i915_ironlake_get_mem_freq(dev_priv); /* For FIFO watermark updates */ - if (INTEL_INFO(dev)->gen >= 9) { - skl_setup_wm_latency(dev); - dev_priv->display.update_wm = skl_update_wm; + if (INTEL_GEN(dev_priv) >= 9) { + skl_setup_wm_latency(dev_priv); + dev_priv->display.initial_watermarks = skl_initial_wm; + dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm; dev_priv->display.compute_global_watermarks = skl_compute_wm; } else if (HAS_PCH_SPLIT(dev_priv)) { - ilk_setup_wm_latency(dev); + ilk_setup_wm_latency(dev_priv); if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] && dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || @@ -7767,12 +7719,12 @@ void intel_init_pm(struct drm_device *dev) "Disable CxSR\n"); } } else if (IS_CHERRYVIEW(dev_priv)) { - vlv_setup_wm_latency(dev); + vlv_setup_wm_latency(dev_priv); dev_priv->display.update_wm = vlv_update_wm; } else if (IS_VALLEYVIEW(dev_priv)) { - vlv_setup_wm_latency(dev); + vlv_setup_wm_latency(dev_priv); dev_priv->display.update_wm = vlv_update_wm; - } else if (IS_PINEVIEW(dev)) { + } else if (IS_PINEVIEW(dev_priv)) { if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv), dev_priv->is_ddr3, dev_priv->fsb_freq, @@ -7795,7 +7747,7 @@ void intel_init_pm(struct drm_device *dev) dev_priv->display.update_wm = i9xx_update_wm; dev_priv->display.get_fifo_size = i9xx_get_fifo_size; } else if (IS_GEN2(dev_priv)) { - if (INTEL_INFO(dev)->num_pipes == 1) { + if (INTEL_INFO(dev_priv)->num_pipes == 1) { dev_priv->display.update_wm = i845_update_wm; dev_priv->display.get_fifo_size = i845_get_fifo_size; } else { @@ -8043,5 +7995,4 @@ void intel_pm_setup(struct drm_device *dev) dev_priv->pm.suspended = false; atomic_set(&dev_priv->pm.wakeref_count, 0); - atomic_set(&dev_priv->pm.atomic_seq, 0); } diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c index 271a3e29ff23..7b488e2793d9 100644 --- a/drivers/gpu/drm/i915/intel_psr.c +++ b/drivers/gpu/drm/i915/intel_psr.c @@ -427,7 +427,7 @@ void intel_psr_enable(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); - if (!HAS_PSR(dev)) { + if (!HAS_PSR(dev_priv)) { DRM_DEBUG_KMS("PSR not supported on this platform\n"); return; } @@ -472,7 +472,7 @@ void intel_psr_enable(struct intel_dp *intel_dp) /* Enable PSR on the panel */ hsw_psr_enable_sink(intel_dp); - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) intel_psr_activate(intel_dp); } else { vlv_psr_setup_vsc(intel_dp); @@ -498,7 +498,7 @@ void intel_psr_enable(struct intel_dp *intel_dp) * - On HSW/BDW we get a recoverable frozen screen until next * exit-activate sequence. */ - if (INTEL_INFO(dev)->gen < 9) + if (INTEL_GEN(dev_priv) < 9) schedule_delayed_work(&dev_priv->psr.work, msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5)); diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 32786ba199b9..aeb637dc1fdf 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -648,7 +648,7 @@ static int intel_rcs_ctx_init(struct drm_i915_gem_request *req) if (ret != 0) return ret; - ret = i915_gem_render_state_init(req); + ret = i915_gem_render_state_emit(req); if (ret) return ret; @@ -1213,90 +1213,62 @@ static void render_ring_cleanup(struct intel_engine_cs *engine) i915_vma_unpin_and_release(&dev_priv->semaphore); } -static int gen8_rcs_signal(struct drm_i915_gem_request *req) +static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *out) { - struct intel_ring *ring = req->ring; struct drm_i915_private *dev_priv = req->i915; struct intel_engine_cs *waiter; enum intel_engine_id id; - int ret, num_rings; - - num_rings = INTEL_INFO(dev_priv)->num_rings; - ret = intel_ring_begin(req, (num_rings-1) * 8); - if (ret) - return ret; for_each_engine(waiter, dev_priv, id) { u64 gtt_offset = req->engine->semaphore.signal_ggtt[id]; if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(ring, - PIPE_CONTROL_GLOBAL_GTT_IVB | - PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_CS_STALL); - intel_ring_emit(ring, lower_32_bits(gtt_offset)); - intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, - MI_SEMAPHORE_SIGNAL | - MI_SEMAPHORE_TARGET(waiter->hw_id)); - intel_ring_emit(ring, 0); + *out++ = GFX_OP_PIPE_CONTROL(6); + *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_CS_STALL); + *out++ = lower_32_bits(gtt_offset); + *out++ = upper_32_bits(gtt_offset); + *out++ = req->global_seqno; + *out++ = 0; + *out++ = (MI_SEMAPHORE_SIGNAL | + MI_SEMAPHORE_TARGET(waiter->hw_id)); + *out++ = 0; } - intel_ring_advance(ring); - return 0; + return out; } -static int gen8_xcs_signal(struct drm_i915_gem_request *req) +static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *out) { - struct intel_ring *ring = req->ring; struct drm_i915_private *dev_priv = req->i915; struct intel_engine_cs *waiter; enum intel_engine_id id; - int ret, num_rings; - - num_rings = INTEL_INFO(dev_priv)->num_rings; - ret = intel_ring_begin(req, (num_rings-1) * 6); - if (ret) - return ret; for_each_engine(waiter, dev_priv, id) { u64 gtt_offset = req->engine->semaphore.signal_ggtt[id]; if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - intel_ring_emit(ring, - (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); - intel_ring_emit(ring, - lower_32_bits(gtt_offset) | - MI_FLUSH_DW_USE_GTT); - intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); - intel_ring_emit(ring, - MI_SEMAPHORE_SIGNAL | - MI_SEMAPHORE_TARGET(waiter->hw_id)); - intel_ring_emit(ring, 0); + *out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW; + *out++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT; + *out++ = upper_32_bits(gtt_offset); + *out++ = req->global_seqno; + *out++ = (MI_SEMAPHORE_SIGNAL | + MI_SEMAPHORE_TARGET(waiter->hw_id)); + *out++ = 0; } - intel_ring_advance(ring); - return 0; + return out; } -static int gen6_signal(struct drm_i915_gem_request *req) +static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *out) { - struct intel_ring *ring = req->ring; struct drm_i915_private *dev_priv = req->i915; struct intel_engine_cs *engine; enum intel_engine_id id; - int ret, num_rings; - - num_rings = INTEL_INFO(dev_priv)->num_rings; - ret = intel_ring_begin(req, round_up((num_rings-1) * 3, 2)); - if (ret) - return ret; + int num_rings = 0; for_each_engine(engine, dev_priv, id) { i915_reg_t mbox_reg; @@ -1306,101 +1278,80 @@ static int gen6_signal(struct drm_i915_gem_request *req) mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id]; if (i915_mmio_reg_valid(mbox_reg)) { - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit_reg(ring, mbox_reg); - intel_ring_emit(ring, req->fence.seqno); + *out++ = MI_LOAD_REGISTER_IMM(1); + *out++ = i915_mmio_reg_offset(mbox_reg); + *out++ = req->global_seqno; + num_rings++; } } + if (num_rings & 1) + *out++ = MI_NOOP; - /* If num_dwords was rounded, make sure the tail pointer is correct */ - if (num_rings % 2 == 0) - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); - - return 0; + return out; } static void i9xx_submit_request(struct drm_i915_gem_request *request) { struct drm_i915_private *dev_priv = request->i915; - I915_WRITE_TAIL(request->engine, - intel_ring_offset(request->ring, request->tail)); + i915_gem_request_submit(request); + + I915_WRITE_TAIL(request->engine, request->tail); } -static int i9xx_emit_request(struct drm_i915_gem_request *req) +static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req, + u32 *out) { - struct intel_ring *ring = req->ring; - int ret; + *out++ = MI_STORE_DWORD_INDEX; + *out++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT; + *out++ = req->global_seqno; + *out++ = MI_USER_INTERRUPT; - ret = intel_ring_begin(req, 4); - if (ret) - return ret; - - intel_ring_emit(ring, MI_STORE_DWORD_INDEX); - intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, req->fence.seqno); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_advance(ring); - - req->tail = ring->tail; - - return 0; + req->tail = intel_ring_offset(req->ring, out); } +static const int i9xx_emit_breadcrumb_sz = 4; + /** - * gen6_sema_emit_request - Update the semaphore mailbox registers + * gen6_sema_emit_breadcrumb - Update the semaphore mailbox registers * * @request - request to write to the ring * * Update the mailbox registers in the *other* rings with the current seqno. * This acts like a signal in the canonical semaphore. */ -static int gen6_sema_emit_request(struct drm_i915_gem_request *req) +static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req, + u32 *out) { - int ret; - - ret = req->engine->semaphore.signal(req); - if (ret) - return ret; - - return i9xx_emit_request(req); + return i9xx_emit_breadcrumb(req, + req->engine->semaphore.signal(req, out)); } -static int gen8_render_emit_request(struct drm_i915_gem_request *req) +static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req, + u32 *out) { struct intel_engine_cs *engine = req->engine; - struct intel_ring *ring = req->ring; - int ret; - if (engine->semaphore.signal) { - ret = engine->semaphore.signal(req); - if (ret) - return ret; - } + if (engine->semaphore.signal) + out = engine->semaphore.signal(req, out); - ret = intel_ring_begin(req, 8); - if (ret) - return ret; - - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(ring, (PIPE_CONTROL_GLOBAL_GTT_IVB | + *out++ = GFX_OP_PIPE_CONTROL(6); + *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_QW_WRITE)); - intel_ring_emit(ring, intel_hws_seqno_address(engine)); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(req)); + PIPE_CONTROL_QW_WRITE); + *out++ = intel_hws_seqno_address(engine); + *out++ = 0; + *out++ = req->global_seqno; /* We're thrashing one dword of HWS. */ - intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); - - req->tail = ring->tail; + *out++ = 0; + *out++ = MI_USER_INTERRUPT; + *out++ = MI_NOOP; - return 0; + req->tail = intel_ring_offset(req->ring, out); } +static const int gen8_render_emit_breadcrumb_sz = 8; + /** * intel_ring_sync - sync the waiter to the signaller on seqno * @@ -1427,7 +1378,7 @@ gen8_ring_sync_to(struct drm_i915_gem_request *req, MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT | MI_SEMAPHORE_SAD_GTE_SDD); - intel_ring_emit(ring, signal->fence.seqno); + intel_ring_emit(ring, signal->global_seqno); intel_ring_emit(ring, lower_32_bits(offset)); intel_ring_emit(ring, upper_32_bits(offset)); intel_ring_advance(ring); @@ -1465,7 +1416,7 @@ gen6_ring_sync_to(struct drm_i915_gem_request *req, * seqno is >= the last seqno executed. However for hardware the * comparison is strictly greater than. */ - intel_ring_emit(ring, signal->fence.seqno - 1); + intel_ring_emit(ring, signal->global_seqno - 1); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); @@ -1608,7 +1559,7 @@ hsw_vebox_irq_enable(struct intel_engine_cs *engine) struct drm_i915_private *dev_priv = engine->i915; I915_WRITE_IMR(engine, ~engine->irq_enable_mask); - gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask); + gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask); } static void @@ -1617,7 +1568,7 @@ hsw_vebox_irq_disable(struct intel_engine_cs *engine) struct drm_i915_private *dev_priv = engine->i915; I915_WRITE_IMR(engine, ~0); - gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask); + gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask); } static void @@ -1762,14 +1713,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine) static void cleanup_status_page(struct intel_engine_cs *engine) { struct i915_vma *vma; + struct drm_i915_gem_object *obj; vma = fetch_and_zero(&engine->status_page.vma); if (!vma) return; + obj = vma->obj; + i915_vma_unpin(vma); - i915_gem_object_unpin_map(vma->obj); - i915_vma_put(vma); + i915_vma_close(vma); + + i915_gem_object_unpin_map(obj); + __i915_gem_object_release_unless_active(obj); } static int init_status_page(struct intel_engine_cs *engine) @@ -1777,9 +1733,10 @@ static int init_status_page(struct intel_engine_cs *engine) struct drm_i915_gem_object *obj; struct i915_vma *vma; unsigned int flags; + void *vaddr; int ret; - obj = i915_gem_object_create(&engine->i915->drm, 4096); + obj = i915_gem_object_create_internal(engine->i915, 4096); if (IS_ERR(obj)) { DRM_ERROR("Failed to allocate status page\n"); return PTR_ERR(obj); @@ -1812,15 +1769,22 @@ static int init_status_page(struct intel_engine_cs *engine) if (ret) goto err; + vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(vaddr)) { + ret = PTR_ERR(vaddr); + goto err_unpin; + } + engine->status_page.vma = vma; engine->status_page.ggtt_offset = i915_ggtt_offset(vma); - engine->status_page.page_addr = - i915_gem_object_pin_map(obj, I915_MAP_WB); + engine->status_page.page_addr = memset(vaddr, 0, 4096); DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", engine->name, i915_ggtt_offset(vma)); return 0; +err_unpin: + i915_vma_unpin(vma); err: i915_gem_object_put(obj); return ret; @@ -1967,7 +1931,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size) void intel_ring_free(struct intel_ring *ring) { - i915_vma_put(ring->vma); + struct drm_i915_gem_object *obj = ring->vma->obj; + + i915_vma_close(ring->vma); + __i915_gem_object_release_unless_active(obj); + kfree(ring); } @@ -1983,14 +1951,13 @@ static int intel_ring_context_pin(struct i915_gem_context *ctx, return 0; if (ce->state) { - ret = i915_gem_object_set_to_gtt_domain(ce->state->obj, false); - if (ret) - goto error; + struct i915_vma *vma; - ret = i915_vma_pin(ce->state, 0, ctx->ggtt_alignment, - PIN_GLOBAL | PIN_HIGH); - if (ret) + vma = i915_gem_context_pin_legacy(ctx, PIN_HIGH); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); goto error; + } } /* The kernel context is only used as a placeholder for flushing the @@ -2037,9 +2004,6 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine) intel_engine_setup_common(engine); - memset(engine->semaphore.sync_seqno, 0, - sizeof(engine->semaphore.sync_seqno)); - ret = intel_engine_init_common(engine); if (ret) goto error; @@ -2155,7 +2119,9 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes) { struct intel_ring *ring = req->ring; struct drm_i915_gem_request *target; - int ret; + long timeout; + + lockdep_assert_held(&req->i915->drm.struct_mutex); intel_ring_update_space(ring); if (ring->space >= bytes) @@ -2185,11 +2151,11 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes) if (WARN_ON(&target->ring_link == &ring->request_list)) return -ENOSPC; - ret = i915_wait_request(target, - I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, - NULL, NO_WAITBOOST); - if (ret) - return ret; + timeout = i915_wait_request(target, + I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT); + if (timeout < 0) + return timeout; i915_gem_request_retire_upto(target); @@ -2618,9 +2584,22 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv, engine->init_hw = init_ring_common; engine->reset_hw = reset_ring_common; - engine->emit_request = i9xx_emit_request; - if (i915.semaphores) - engine->emit_request = gen6_sema_emit_request; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz; + if (i915.semaphores) { + int num_rings; + + engine->emit_breadcrumb = gen6_sema_emit_breadcrumb; + + num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1; + if (INTEL_GEN(dev_priv) >= 8) { + engine->emit_breadcrumb_sz += num_rings * 6; + } else { + engine->emit_breadcrumb_sz += num_rings * 3; + if (num_rings & 1) + engine->emit_breadcrumb_sz++; + } + } engine->submit_request = i9xx_submit_request; if (INTEL_GEN(dev_priv) >= 8) @@ -2647,10 +2626,18 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine) if (INTEL_GEN(dev_priv) >= 8) { engine->init_context = intel_rcs_ctx_init; - engine->emit_request = gen8_render_emit_request; + engine->emit_breadcrumb = gen8_render_emit_breadcrumb; + engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz; engine->emit_flush = gen8_render_ring_flush; - if (i915.semaphores) + if (i915.semaphores) { + int num_rings; + engine->semaphore.signal = gen8_rcs_signal; + + num_rings = + hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1; + engine->emit_breadcrumb_sz += num_rings * 6; + } } else if (INTEL_GEN(dev_priv) >= 6) { engine->init_context = intel_rcs_ctx_init; engine->emit_flush = gen7_render_ring_flush; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 32b2e6332ccf..3466b4e77e7c 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -4,6 +4,7 @@ #include <linux/hashtable.h> #include "i915_gem_batch_pool.h" #include "i915_gem_request.h" +#include "i915_gem_timeline.h" #define I915_CMD_HASH_ORDER 9 @@ -157,6 +158,7 @@ struct i915_ctx_workarounds { }; struct drm_i915_gem_request; +struct intel_render_state; struct intel_engine_cs { struct drm_i915_private *i915; @@ -168,7 +170,6 @@ struct intel_engine_cs { VCS2, /* Keep instances of the same type engine together. */ VECS } id; -#define I915_NUM_ENGINES 5 #define _VCS(n) (VCS + (n)) unsigned int exec_id; enum intel_engine_hw_id { @@ -179,10 +180,12 @@ struct intel_engine_cs { VCS2_HW } hw_id; enum intel_engine_hw_id guc_id; /* XXX same as hw_id? */ - u64 fence_context; u32 mmio_base; unsigned int irq_shift; struct intel_ring *buffer; + struct intel_timeline *timeline; + + struct intel_render_state *render_state; /* Rather than have every client wait upon all user interrupts, * with the herd waking after every interrupt and each doing the @@ -204,7 +207,7 @@ struct intel_engine_cs { struct task_struct __rcu *irq_seqno_bh; /* bh for interrupts */ bool irq_posted; - spinlock_t lock; /* protects the lists of requests */ + spinlock_t lock; /* protects the lists of requests; irqsafe */ struct rb_root waiters; /* sorted by retirement, priority */ struct rb_root signals; /* sorted by retirement */ struct intel_wait *first_wait; /* oldest waiter by retirement */ @@ -252,7 +255,9 @@ struct intel_engine_cs { #define I915_DISPATCH_SECURE BIT(0) #define I915_DISPATCH_PINNED BIT(1) #define I915_DISPATCH_RS BIT(2) - int (*emit_request)(struct drm_i915_gem_request *req); + void (*emit_breadcrumb)(struct drm_i915_gem_request *req, + u32 *out); + int emit_breadcrumb_sz; /* Pass the request to the hardware queue (e.g. directly into * the legacy ringbuffer or to the end of an execlist). @@ -262,6 +267,15 @@ struct intel_engine_cs { */ void (*submit_request)(struct drm_i915_gem_request *req); + /* Call when the priority on a request has changed and it and its + * dependencies may need rescheduling. Note the request itself may + * not be ready to run! + * + * Called under the struct_mutex. + */ + void (*schedule)(struct drm_i915_gem_request *request, + int priority); + /* Some chipsets are not quite as coherent as advertised and need * an expensive kick to force a true read of the up-to-date seqno. * However, the up-to-date seqno is not always required and the last @@ -309,8 +323,6 @@ struct intel_engine_cs { * ie. transpose of f(x, y) */ struct { - u32 sync_seqno[I915_NUM_ENGINES-1]; - union { #define GEN6_SEMAPHORE_LAST VECS_HW #define GEN6_NUM_SEMAPHORES (GEN6_SEMAPHORE_LAST + 1) @@ -327,43 +339,22 @@ struct intel_engine_cs { /* AKA wait() */ int (*sync_to)(struct drm_i915_gem_request *req, struct drm_i915_gem_request *signal); - int (*signal)(struct drm_i915_gem_request *req); + u32 *(*signal)(struct drm_i915_gem_request *req, u32 *out); } semaphore; /* Execlists */ struct tasklet_struct irq_tasklet; - spinlock_t execlist_lock; /* used inside tasklet, use spin_lock_bh */ struct execlist_port { struct drm_i915_gem_request *request; unsigned int count; } execlist_port[2]; - struct list_head execlist_queue; + struct rb_root execlist_queue; + struct rb_node *execlist_first; unsigned int fw_domains; bool disable_lite_restore_wa; bool preempt_wa; u32 ctx_desc_template; - /** - * List of breadcrumbs associated with GPU requests currently - * outstanding. - */ - struct list_head request_list; - - /** - * Seqno of request most recently submitted to request_list. - * Used exclusively by hang checker to avoid grabbing lock while - * inspecting request list. - */ - u32 last_submitted_seqno; - u32 last_pending_seqno; - - /* An RCU guarded pointer to the last request. No reference is - * held to the request, users must carefully acquire a reference to - * the request using i915_gem_active_get_rcu(), or hold the - * struct_mutex. - */ - struct i915_gem_active last_request; - struct i915_gem_context *last_context; struct intel_engine_hangcheck hangcheck; @@ -401,27 +392,6 @@ intel_engine_flag(const struct intel_engine_cs *engine) return 1 << engine->id; } -static inline u32 -intel_engine_sync_index(struct intel_engine_cs *engine, - struct intel_engine_cs *other) -{ - int idx; - - /* - * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2; - * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs; - * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs; - * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs; - * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs; - */ - - idx = (other->id - engine->id) - 1; - if (idx < 0) - idx += I915_NUM_ENGINES; - - return idx; -} - static inline void intel_flush_status_page(struct intel_engine_cs *engine, int reg) { @@ -504,30 +474,23 @@ static inline void intel_ring_advance(struct intel_ring *ring) */ } -static inline u32 intel_ring_offset(struct intel_ring *ring, u32 value) +static inline u32 intel_ring_offset(struct intel_ring *ring, void *addr) { /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */ - return value & (ring->size - 1); + u32 offset = addr - ring->vaddr; + return offset & (ring->size - 1); } int __intel_ring_space(int head, int tail, int size); void intel_ring_update_space(struct intel_ring *ring); -void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno); +void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno); void intel_engine_setup_common(struct intel_engine_cs *engine); int intel_engine_init_common(struct intel_engine_cs *engine); int intel_engine_create_scratch(struct intel_engine_cs *engine, int size); void intel_engine_cleanup_common(struct intel_engine_cs *engine); -static inline int intel_engine_idle(struct intel_engine_cs *engine, - unsigned int flags) -{ - /* Wait upon the last request to be completed */ - return i915_gem_active_wait_unlocked(&engine->last_request, - flags, NULL, NULL); -} - int intel_init_render_ring_buffer(struct intel_engine_cs *engine); int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine); int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine); @@ -542,6 +505,18 @@ static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine) return intel_read_status_page(engine, I915_GEM_HWS_INDEX); } +static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine) +{ + /* We are only peeking at the tail of the submit queue (and not the + * queue itself) in order to gain a hint as to the current active + * state of the engine. Callers are not expected to be taking + * engine->timeline->lock, nor are they expected to be concerned + * wtih serialising this hint with anything, so document it as + * a hint and nothing more. + */ + return READ_ONCE(engine->timeline->last_submitted_seqno); +} + int init_workarounds_ring(struct intel_engine_cs *engine); void intel_engine_get_instdone(struct intel_engine_cs *engine, @@ -612,12 +587,6 @@ static inline bool intel_engine_wakeup(const struct intel_engine_cs *engine) void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine); void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); -unsigned int intel_kick_waiters(struct drm_i915_private *i915); -unsigned int intel_kick_signalers(struct drm_i915_private *i915); - -static inline bool intel_engine_is_active(struct intel_engine_cs *engine) -{ - return i915_gem_active_isset(&engine->last_request); -} +unsigned int intel_breadcrumbs_busy(struct drm_i915_private *i915); #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index ee56a8756c07..356c662ad453 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -330,7 +330,7 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv, * sure vgacon can keep working normally without triggering interrupts * and error messages. */ - if (power_well->data == SKL_DISP_PW_2) { + if (power_well->id == SKL_DISP_PW_2) { vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO); outb(inb(VGA_MSR_READ), VGA_MSR_WRITE); vga_put(pdev, VGA_RSRC_LEGACY_IO); @@ -343,7 +343,7 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv, static void skl_power_well_pre_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - if (power_well->data == SKL_DISP_PW_2) + if (power_well->id == SKL_DISP_PW_2) gen8_irq_power_well_pre_disable(dev_priv, 1 << PIPE_C | 1 << PIPE_B); } @@ -658,7 +658,7 @@ static void gen9_sanitize_power_well_requests(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum skl_disp_power_wells power_well_id = power_well->data; + enum skl_disp_power_wells power_well_id = power_well->id; u32 val; u32 mask; @@ -703,7 +703,7 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, tmp = I915_READ(HSW_PWR_WELL_DRIVER); fuse_status = I915_READ(SKL_FUSE_STATUS); - switch (power_well->data) { + switch (power_well->id) { case SKL_DISP_PW_1: if (intel_wait_for_register(dev_priv, SKL_FUSE_STATUS, @@ -727,13 +727,13 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, case SKL_DISP_PW_MISC_IO: break; default: - WARN(1, "Unknown power well %lu\n", power_well->data); + WARN(1, "Unknown power well %lu\n", power_well->id); return; } - req_mask = SKL_POWER_WELL_REQ(power_well->data); + req_mask = SKL_POWER_WELL_REQ(power_well->id); enable_requested = tmp & req_mask; - state_mask = SKL_POWER_WELL_STATE(power_well->data); + state_mask = SKL_POWER_WELL_STATE(power_well->id); is_enabled = tmp & state_mask; if (!enable && enable_requested) @@ -769,14 +769,14 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, power_well->name, enable ? "enable" : "disable"); if (check_fuse_status) { - if (power_well->data == SKL_DISP_PW_1) { + if (power_well->id == SKL_DISP_PW_1) { if (intel_wait_for_register(dev_priv, SKL_FUSE_STATUS, SKL_FUSE_PG1_DIST_STATUS, SKL_FUSE_PG1_DIST_STATUS, 1)) DRM_ERROR("PG1 distributing status timeout\n"); - } else if (power_well->data == SKL_DISP_PW_2) { + } else if (power_well->id == SKL_DISP_PW_2) { if (intel_wait_for_register(dev_priv, SKL_FUSE_STATUS, SKL_FUSE_PG2_DIST_STATUS, @@ -818,8 +818,8 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv, static bool skl_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - uint32_t mask = SKL_POWER_WELL_REQ(power_well->data) | - SKL_POWER_WELL_STATE(power_well->data); + uint32_t mask = SKL_POWER_WELL_REQ(power_well->id) | + SKL_POWER_WELL_STATE(power_well->id); return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask; } @@ -845,45 +845,22 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv, skl_set_power_well(dev_priv, power_well, false); } -static enum dpio_phy bxt_power_well_to_phy(struct i915_power_well *power_well) -{ - enum skl_disp_power_wells power_well_id = power_well->data; - - return power_well_id == BXT_DPIO_CMN_A ? DPIO_PHY1 : DPIO_PHY0; -} - static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum skl_disp_power_wells power_well_id = power_well->data; - struct i915_power_well *cmn_a_well = NULL; - - if (power_well_id == BXT_DPIO_CMN_BC) { - /* - * We need to copy the GRC calibration value from the eDP PHY, - * so make sure it's powered up. - */ - cmn_a_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A); - intel_power_well_get(dev_priv, cmn_a_well); - } - - bxt_ddi_phy_init(dev_priv, bxt_power_well_to_phy(power_well)); - - if (cmn_a_well) - intel_power_well_put(dev_priv, cmn_a_well); + bxt_ddi_phy_init(dev_priv, power_well->data); } static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - bxt_ddi_phy_uninit(dev_priv, bxt_power_well_to_phy(power_well)); + bxt_ddi_phy_uninit(dev_priv, power_well->data); } static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - return bxt_ddi_phy_is_enabled(dev_priv, - bxt_power_well_to_phy(power_well)); + return bxt_ddi_phy_is_enabled(dev_priv, power_well->data); } static void bxt_dpio_cmn_power_well_sync_hw(struct drm_i915_private *dev_priv, @@ -902,13 +879,11 @@ static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv) power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A); if (power_well->count > 0) - bxt_ddi_phy_verify_state(dev_priv, - bxt_power_well_to_phy(power_well)); + bxt_ddi_phy_verify_state(dev_priv, power_well->data); power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC); if (power_well->count > 0) - bxt_ddi_phy_verify_state(dev_priv, - bxt_power_well_to_phy(power_well)); + bxt_ddi_phy_verify_state(dev_priv, power_well->data); } static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv, @@ -932,7 +907,7 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv, gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); WARN_ON(dev_priv->cdclk_freq != - dev_priv->display.get_display_clock_speed(&dev_priv->drm)); + dev_priv->display.get_display_clock_speed(dev_priv)); gen9_assert_dbuf_enabled(dev_priv); @@ -975,7 +950,7 @@ static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv, static void vlv_set_power_well(struct drm_i915_private *dev_priv, struct i915_power_well *power_well, bool enable) { - enum punit_power_well power_well_id = power_well->data; + enum punit_power_well power_well_id = power_well->id; u32 mask; u32 state; u32 ctrl; @@ -1029,7 +1004,7 @@ static void vlv_power_well_disable(struct drm_i915_private *dev_priv, static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - int power_well_id = power_well->data; + int power_well_id = power_well->id; bool enabled = false; u32 mask; u32 state; @@ -1091,7 +1066,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv) * * CHV DPLL B/C have some issues if VGA mode is enabled. */ - for_each_pipe(&dev_priv->drm, pipe) { + for_each_pipe(dev_priv, pipe) { u32 val = I915_READ(DPLL(pipe)); val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; @@ -1122,7 +1097,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv) intel_crt_reset(&encoder->base); } - i915_redisable_vga_power_on(&dev_priv->drm); + i915_redisable_vga_power_on(dev_priv); intel_pps_unlock_regs_wa(dev_priv); } @@ -1138,13 +1113,15 @@ static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv) intel_power_sequencer_reset(dev_priv); - intel_hpd_poll_init(dev_priv); + /* Prevent us from re-enabling polling on accident in late suspend */ + if (!dev_priv->drm.dev->power.is_suspended) + intel_hpd_poll_init(dev_priv); } static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D); vlv_set_power_well(dev_priv, power_well, true); @@ -1154,7 +1131,7 @@ static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D); vlv_display_power_well_deinit(dev_priv); @@ -1164,7 +1141,7 @@ static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC); /* since ref/cri clock was enabled */ udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ @@ -1190,7 +1167,7 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, { enum pipe pipe; - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC); for_each_pipe(dev_priv, pipe) assert_pll_disabled(dev_priv, pipe); @@ -1213,7 +1190,7 @@ static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_pr struct i915_power_well *power_well; power_well = &power_domains->power_wells[i]; - if (power_well->data == power_well_id) + if (power_well->id == power_well_id) return power_well; } @@ -1337,10 +1314,10 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, enum pipe pipe; uint32_t tmp; - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && - power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC && + power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D); - if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) { pipe = PIPE_A; phy = DPIO_PHY0; } else { @@ -1368,7 +1345,7 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, DPIO_SUS_CLK_CONFIG_GATE_CLKREQ; vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp); - if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) { tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1); tmp |= DPIO_DYNPWRDOWNEN_CH1; vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp); @@ -1399,10 +1376,10 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, { enum dpio_phy phy; - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && - power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC && + power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D); - if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) { phy = DPIO_PHY0; assert_pll_disabled(dev_priv, PIPE_A); assert_pll_disabled(dev_priv, PIPE_B); @@ -1551,7 +1528,7 @@ void chv_phy_powergate_lanes(struct intel_encoder *encoder, static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum pipe pipe = power_well->data; + enum pipe pipe = power_well->id; bool enabled; u32 state, ctrl; @@ -1581,7 +1558,7 @@ static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv, struct i915_power_well *power_well, bool enable) { - enum pipe pipe = power_well->data; + enum pipe pipe = power_well->id; u32 state; u32 ctrl; @@ -1614,7 +1591,7 @@ out: static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A); + WARN_ON_ONCE(power_well->id != PIPE_A); chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0); } @@ -1622,7 +1599,7 @@ static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A); + WARN_ON_ONCE(power_well->id != PIPE_A); chv_set_pipe_power_well(dev_priv, power_well, true); @@ -1632,7 +1609,7 @@ static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A); + WARN_ON_ONCE(power_well->id != PIPE_A); vlv_display_power_well_deinit(dev_priv); @@ -1976,12 +1953,12 @@ static struct i915_power_well vlv_power_wells[] = { .always_on = 1, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, - .data = PUNIT_POWER_WELL_ALWAYS_ON, + .id = PUNIT_POWER_WELL_ALWAYS_ON, }, { .name = "display", .domains = VLV_DISPLAY_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DISP2D, + .id = PUNIT_POWER_WELL_DISP2D, .ops = &vlv_display_power_well_ops, }, { @@ -1991,7 +1968,7 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, + .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, }, { .name = "dpio-tx-b-23", @@ -2000,7 +1977,7 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, + .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, }, { .name = "dpio-tx-c-01", @@ -2009,7 +1986,7 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, + .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, }, { .name = "dpio-tx-c-23", @@ -2018,12 +1995,12 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, + .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, }, { .name = "dpio-common", .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DPIO_CMN_BC, + .id = PUNIT_POWER_WELL_DPIO_CMN_BC, .ops = &vlv_dpio_cmn_power_well_ops, }, }; @@ -2043,19 +2020,19 @@ static struct i915_power_well chv_power_wells[] = { * required for any pipe to work. */ .domains = CHV_DISPLAY_POWER_DOMAINS, - .data = PIPE_A, + .id = PIPE_A, .ops = &chv_pipe_power_well_ops, }, { .name = "dpio-common-bc", .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DPIO_CMN_BC, + .id = PUNIT_POWER_WELL_DPIO_CMN_BC, .ops = &chv_dpio_cmn_power_well_ops, }, { .name = "dpio-common-d", .domains = CHV_DPIO_CMN_D_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DPIO_CMN_D, + .id = PUNIT_POWER_WELL_DPIO_CMN_D, .ops = &chv_dpio_cmn_power_well_ops, }, }; @@ -2078,57 +2055,57 @@ static struct i915_power_well skl_power_wells[] = { .always_on = 1, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, - .data = SKL_DISP_PW_ALWAYS_ON, + .id = SKL_DISP_PW_ALWAYS_ON, }, { .name = "power well 1", /* Handled by the DMC firmware */ .domains = 0, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_1, + .id = SKL_DISP_PW_1, }, { .name = "MISC IO power well", /* Handled by the DMC firmware */ .domains = 0, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_MISC_IO, + .id = SKL_DISP_PW_MISC_IO, }, { .name = "DC off", .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS, .ops = &gen9_dc_off_power_well_ops, - .data = SKL_DISP_PW_DC_OFF, + .id = SKL_DISP_PW_DC_OFF, }, { .name = "power well 2", .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_2, + .id = SKL_DISP_PW_2, }, { .name = "DDI A/E power well", .domains = SKL_DISPLAY_DDI_A_E_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_A_E, + .id = SKL_DISP_PW_DDI_A_E, }, { .name = "DDI B power well", .domains = SKL_DISPLAY_DDI_B_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_B, + .id = SKL_DISP_PW_DDI_B, }, { .name = "DDI C power well", .domains = SKL_DISPLAY_DDI_C_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_C, + .id = SKL_DISP_PW_DDI_C, }, { .name = "DDI D power well", .domains = SKL_DISPLAY_DDI_D_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_D, + .id = SKL_DISP_PW_DDI_D, }, }; @@ -2143,31 +2120,33 @@ static struct i915_power_well bxt_power_wells[] = { .name = "power well 1", .domains = 0, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_1, + .id = SKL_DISP_PW_1, }, { .name = "DC off", .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS, .ops = &gen9_dc_off_power_well_ops, - .data = SKL_DISP_PW_DC_OFF, + .id = SKL_DISP_PW_DC_OFF, }, { .name = "power well 2", .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_2, + .id = SKL_DISP_PW_2, }, { .name = "dpio-common-a", .domains = BXT_DPIO_CMN_A_POWER_DOMAINS, .ops = &bxt_dpio_cmn_power_well_ops, - .data = BXT_DPIO_CMN_A, + .id = BXT_DPIO_CMN_A, + .data = DPIO_PHY1, }, { .name = "dpio-common-bc", .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS, .ops = &bxt_dpio_cmn_power_well_ops, - .data = BXT_DPIO_CMN_BC, + .id = BXT_DPIO_CMN_BC, + .data = DPIO_PHY0, }, }; @@ -2736,8 +2715,7 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv) struct device *kdev = &pdev->dev; assert_rpm_wakelock_held(dev_priv); - if (atomic_dec_and_test(&dev_priv->pm.wakeref_count)) - atomic_inc(&dev_priv->pm.atomic_seq); + atomic_dec(&dev_priv->pm.wakeref_count); pm_runtime_mark_last_busy(kdev); pm_runtime_put_autosuspend(kdev); diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index 49fb95d03d74..27808e91cb5a 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -1195,8 +1195,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder, struct intel_crtc_state *crtc_state, struct drm_connector_state *conn_state) { - struct drm_device *dev = intel_encoder->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev); struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; struct drm_display_mode *mode = &crtc_state->base.mode; @@ -1269,13 +1268,13 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder, return; /* Set the SDVO control regs. */ - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { /* The real mode polarity is set by the SDVO commands, using * struct intel_sdvo_dtd. */ sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH; if (!HAS_PCH_SPLIT(dev_priv) && crtc_state->limited_color_range) sdvox |= HDMI_COLOR_RANGE_16_235; - if (INTEL_INFO(dev)->gen < 5) + if (INTEL_GEN(dev_priv) < 5) sdvox |= SDVO_BORDER_ENABLE; } else { sdvox = I915_READ(intel_sdvo->sdvo_reg); @@ -1294,7 +1293,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder, if (intel_sdvo->has_hdmi_audio) sdvox |= SDVO_AUDIO_ENABLE; - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { /* done in crtc_mode_set as the dpll_md reg must be written early */ } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) || IS_G33(dev_priv)) { @@ -1305,7 +1304,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder, } if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL && - INTEL_INFO(dev)->gen < 5) + INTEL_GEN(dev_priv) < 5) sdvox |= SDVO_STALL_SELECT; intel_sdvo_write_sdvox(intel_sdvo, sdvox); } @@ -1472,7 +1471,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder, temp &= ~SDVO_ENABLE; intel_sdvo_write_sdvox(intel_sdvo, temp); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -1509,7 +1508,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder, intel_sdvo_write_sdvox(intel_sdvo, temp); for (i = 0; i < 2; i++) - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); success = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2); /* Warn if the device reported failure to sync. @@ -2411,10 +2410,10 @@ static void intel_sdvo_add_hdmi_properties(struct intel_sdvo *intel_sdvo, struct intel_sdvo_connector *connector) { - struct drm_device *dev = connector->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(connector->base.base.dev); intel_attach_force_audio_property(&connector->base.base); - if (INTEL_INFO(dev)->gen >= 4 && IS_MOBILE(dev)) { + if (INTEL_GEN(dev_priv) >= 4 && IS_MOBILE(dev_priv)) { intel_attach_broadcast_rgb_property(&connector->base.base); intel_sdvo->color_range_auto = true; } diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c index 43d0350856e7..8f131a08d440 100644 --- a/drivers/gpu/drm/i915/intel_sprite.c +++ b/drivers/gpu/drm/i915/intel_sprite.c @@ -203,13 +203,8 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(drm_plane); struct drm_framebuffer *fb = plane_state->base.fb; - const struct skl_wm_values *wm = &dev_priv->wm.skl_results; - struct drm_crtc *crtc = crtc_state->base.crtc; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); const int pipe = intel_plane->pipe; const int plane = intel_plane->plane + 1; - const struct skl_plane_wm *p_wm = - &crtc_state->wm.skl.optimal.planes[plane]; u32 plane_ctl; const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; u32 surf_addr = plane_state->main.offset; @@ -229,13 +224,10 @@ skl_update_plane(struct drm_plane *drm_plane, PLANE_CTL_PIPE_CSC_ENABLE; plane_ctl |= skl_plane_ctl_format(fb->pixel_format); - plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]); + plane_ctl |= skl_plane_ctl_tiling(fb->modifier); plane_ctl |= skl_plane_ctl_rotation(rotation); - if (wm->dirty_pipes & drm_crtc_mask(crtc)) - skl_write_plane_wm(intel_crtc, p_wm, &wm->ddb, plane); - if (key->flags) { I915_WRITE(PLANE_KEYVAL(pipe, plane), key->min_value); I915_WRITE(PLANE_KEYMAX(pipe, plane), key->max_value); @@ -291,19 +283,9 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc) struct drm_device *dev = dplane->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(dplane); - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); const int pipe = intel_plane->pipe; const int plane = intel_plane->plane + 1; - /* - * We only populate skl_results on watermark updates, and if the - * plane's visiblity isn't actually changing neither is its watermarks. - */ - if (!dplane->state->visible) - skl_write_plane_wm(to_intel_crtc(crtc), - &cstate->wm.skl.optimal.planes[plane], - &dev_priv->wm.skl_results.ddb, plane); - I915_WRITE(PLANE_CTL(pipe, plane), 0); I915_WRITE(PLANE_SURF(pipe, plane), 0); @@ -362,7 +344,7 @@ vlv_update_plane(struct drm_plane *dplane, int plane = intel_plane->plane; u32 sprctl; u32 sprsurf_offset, linear_offset; - unsigned int rotation = dplane->state->rotation; + unsigned int rotation = plane_state->base.rotation; const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; int crtc_x = plane_state->base.dst.x1; int crtc_y = plane_state->base.dst.y1; @@ -424,9 +406,15 @@ vlv_update_plane(struct drm_plane *dplane, */ sprctl |= SP_GAMMA_ENABLE; - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (fb->modifier == I915_FORMAT_MOD_X_TILED) sprctl |= SP_TILED; + if (rotation & DRM_ROTATE_180) + sprctl |= SP_ROTATE_180; + + if (rotation & DRM_REFLECT_X) + sprctl |= SP_MIRROR; + /* Sizes are 0 based */ src_w--; src_h--; @@ -436,11 +424,11 @@ vlv_update_plane(struct drm_plane *dplane, intel_add_fb_offsets(&x, &y, plane_state, 0); sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); - if (rotation == DRM_ROTATE_180) { - sprctl |= SP_ROTATE_180; - + if (rotation & DRM_ROTATE_180) { x += src_w; y += src_h; + } else if (rotation & DRM_REFLECT_X) { + x += src_w; } linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); @@ -460,7 +448,7 @@ vlv_update_plane(struct drm_plane *dplane, I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]); I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x); - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (fb->modifier == I915_FORMAT_MOD_X_TILED) I915_WRITE(SPTILEOFF(pipe, plane), (y << 16) | x); else I915_WRITE(SPLINOFF(pipe, plane), linear_offset); @@ -543,9 +531,12 @@ ivb_update_plane(struct drm_plane *plane, */ sprctl |= SPRITE_GAMMA_ENABLE; - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (fb->modifier == I915_FORMAT_MOD_X_TILED) sprctl |= SPRITE_TILED; + if (rotation & DRM_ROTATE_180) + sprctl |= SPRITE_ROTATE_180; + if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) sprctl &= ~SPRITE_TRICKLE_FEED_DISABLE; else @@ -566,14 +557,11 @@ ivb_update_plane(struct drm_plane *plane, intel_add_fb_offsets(&x, &y, plane_state, 0); sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); - if (rotation == DRM_ROTATE_180) { - sprctl |= SPRITE_ROTATE_180; - - /* HSW and BDW does this automagically in hardware */ - if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) { - x += src_w; - y += src_h; - } + /* HSW+ does this automagically in hardware */ + if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv) && + rotation & DRM_ROTATE_180) { + x += src_w; + y += src_h; } linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); @@ -596,7 +584,7 @@ ivb_update_plane(struct drm_plane *plane, * register */ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) I915_WRITE(SPROFFSET(pipe), (y << 16) | x); - else if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + else if (fb->modifier == I915_FORMAT_MOD_X_TILED) I915_WRITE(SPRTILEOFF(pipe), (y << 16) | x); else I915_WRITE(SPRLINOFF(pipe), linear_offset); @@ -681,9 +669,12 @@ ilk_update_plane(struct drm_plane *plane, */ dvscntr |= DVS_GAMMA_ENABLE; - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (fb->modifier == I915_FORMAT_MOD_X_TILED) dvscntr |= DVS_TILED; + if (rotation & DRM_ROTATE_180) + dvscntr |= DVS_ROTATE_180; + if (IS_GEN6(dev_priv)) dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */ @@ -700,9 +691,7 @@ ilk_update_plane(struct drm_plane *plane, intel_add_fb_offsets(&x, &y, plane_state, 0); dvssurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); - if (rotation == DRM_ROTATE_180) { - dvscntr |= DVS_ROTATE_180; - + if (rotation & DRM_ROTATE_180) { x += src_w; y += src_h; } @@ -723,7 +712,7 @@ ilk_update_plane(struct drm_plane *plane, I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]); I915_WRITE(DVSPOS(pipe), (crtc_y << 16) | crtc_x); - if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + if (fb->modifier == I915_FORMAT_MOD_X_TILED) I915_WRITE(DVSTILEOFF(pipe), (y << 16) | x); else I915_WRITE(DVSLINOFF(pipe), linear_offset); @@ -773,15 +762,8 @@ intel_check_sprite_plane(struct drm_plane *plane, bool can_scale; int ret; - src->x1 = state->base.src_x; - src->y1 = state->base.src_y; - src->x2 = state->base.src_x + state->base.src_w; - src->y2 = state->base.src_y + state->base.src_h; - - dst->x1 = state->base.crtc_x; - dst->y1 = state->base.crtc_y; - dst->x2 = state->base.crtc_x + state->base.crtc_w; - dst->y2 = state->base.crtc_y + state->base.crtc_h; + *src = drm_plane_state_src(&state->base); + *dst = drm_plane_state_dest(&state->base); if (!fb) { state->base.visible = false; @@ -1042,10 +1024,10 @@ static uint32_t skl_plane_formats[] = { DRM_FORMAT_VYUY, }; -int -intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) +struct intel_plane * +intel_sprite_plane_create(struct drm_i915_private *dev_priv, + enum pipe pipe, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = NULL; struct intel_plane_state *state = NULL; unsigned long possible_crtcs; @@ -1054,9 +1036,6 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) int num_plane_formats; int ret; - if (INTEL_INFO(dev)->gen < 5) - return -ENODEV; - intel_plane = kzalloc(sizeof(*intel_plane), GFP_KERNEL); if (!intel_plane) { ret = -ENOMEM; @@ -1070,25 +1049,25 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) } intel_plane->base.state = &state->base; - switch (INTEL_INFO(dev)->gen) { - case 5: - case 6: + if (INTEL_GEN(dev_priv) >= 9) { intel_plane->can_scale = true; - intel_plane->max_downscale = 16; - intel_plane->update_plane = ilk_update_plane; - intel_plane->disable_plane = ilk_disable_plane; + state->scaler_id = -1; - if (IS_GEN6(dev_priv)) { - plane_formats = snb_plane_formats; - num_plane_formats = ARRAY_SIZE(snb_plane_formats); - } else { - plane_formats = ilk_plane_formats; - num_plane_formats = ARRAY_SIZE(ilk_plane_formats); - } - break; + intel_plane->update_plane = skl_update_plane; + intel_plane->disable_plane = skl_disable_plane; + + plane_formats = skl_plane_formats; + num_plane_formats = ARRAY_SIZE(skl_plane_formats); + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + intel_plane->can_scale = false; + intel_plane->max_downscale = 1; - case 7: - case 8: + intel_plane->update_plane = vlv_update_plane; + intel_plane->disable_plane = vlv_disable_plane; + + plane_formats = vlv_plane_formats; + num_plane_formats = ARRAY_SIZE(vlv_plane_formats); + } else if (INTEL_GEN(dev_priv) >= 7) { if (IS_IVYBRIDGE(dev_priv)) { intel_plane->can_scale = true; intel_plane->max_downscale = 2; @@ -1097,39 +1076,35 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) intel_plane->max_downscale = 1; } - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - intel_plane->update_plane = vlv_update_plane; - intel_plane->disable_plane = vlv_disable_plane; + intel_plane->update_plane = ivb_update_plane; + intel_plane->disable_plane = ivb_disable_plane; - plane_formats = vlv_plane_formats; - num_plane_formats = ARRAY_SIZE(vlv_plane_formats); - } else { - intel_plane->update_plane = ivb_update_plane; - intel_plane->disable_plane = ivb_disable_plane; + plane_formats = snb_plane_formats; + num_plane_formats = ARRAY_SIZE(snb_plane_formats); + } else { + intel_plane->can_scale = true; + intel_plane->max_downscale = 16; + intel_plane->update_plane = ilk_update_plane; + intel_plane->disable_plane = ilk_disable_plane; + + if (IS_GEN6(dev_priv)) { plane_formats = snb_plane_formats; num_plane_formats = ARRAY_SIZE(snb_plane_formats); + } else { + plane_formats = ilk_plane_formats; + num_plane_formats = ARRAY_SIZE(ilk_plane_formats); } - break; - case 9: - intel_plane->can_scale = true; - intel_plane->update_plane = skl_update_plane; - intel_plane->disable_plane = skl_disable_plane; - state->scaler_id = -1; - - plane_formats = skl_plane_formats; - num_plane_formats = ARRAY_SIZE(skl_plane_formats); - break; - default: - MISSING_CASE(INTEL_INFO(dev)->gen); - ret = -ENODEV; - goto fail; } if (INTEL_GEN(dev_priv) >= 9) { supported_rotations = DRM_ROTATE_0 | DRM_ROTATE_90 | DRM_ROTATE_180 | DRM_ROTATE_270; + } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { + supported_rotations = + DRM_ROTATE_0 | DRM_ROTATE_180 | + DRM_REFLECT_X; } else { supported_rotations = DRM_ROTATE_0 | DRM_ROTATE_180; @@ -1142,15 +1117,15 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) possible_crtcs = (1 << pipe); - if (INTEL_INFO(dev)->gen >= 9) - ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, - &intel_plane_funcs, + if (INTEL_GEN(dev_priv) >= 9) + ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base, + possible_crtcs, &intel_plane_funcs, plane_formats, num_plane_formats, DRM_PLANE_TYPE_OVERLAY, "plane %d%c", plane + 2, pipe_name(pipe)); else - ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base, + possible_crtcs, &intel_plane_funcs, plane_formats, num_plane_formats, DRM_PLANE_TYPE_OVERLAY, "sprite %c", sprite_name(pipe, plane)); @@ -1163,11 +1138,11 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) drm_plane_helper_add(&intel_plane->base, &intel_plane_helper_funcs); - return 0; + return intel_plane; fail: kfree(state); kfree(intel_plane); - return ret; + return ERR_PTR(ret); } diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index 7118fb55f57f..78cdfc6833d6 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -856,7 +856,7 @@ intel_enable_tv(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(dev); /* Prevents vblank waits from timing out in intel_tv_detect_type() */ - intel_wait_for_vblank(encoder->base.dev, + intel_wait_for_vblank(dev_priv, to_intel_crtc(encoder->base.crtc)->pipe); I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE); @@ -1029,8 +1029,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); struct intel_tv *intel_tv = enc_to_tv(encoder); const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv); @@ -1116,7 +1115,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder, set_color_conversion(dev_priv, color_conversion); - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) I915_WRITE(TV_CLR_KNOBS, 0x00404000); else I915_WRITE(TV_CLR_KNOBS, 0x00606000); @@ -1238,7 +1237,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv, I915_WRITE(TV_DAC, tv_dac); POSTING_READ(TV_DAC); - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); type = -1; tv_dac = I915_READ(TV_DAC); @@ -1268,7 +1267,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv, POSTING_READ(TV_CTL); /* For unknown reasons the hw barfs if we don't do this vblank wait. */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); /* Restore interrupt config */ if (connector->polled & DRM_CONNECTOR_POLL_HPD) { diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index e2b188dcf908..d7be0d94ba4d 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -402,6 +402,8 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, bool restore_forcewake) { + struct intel_device_info *info = mkwrite_device_info(dev_priv); + /* clear out unclaimed reg detection bit */ if (check_for_unclaimed_mmio(dev_priv)) DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n"); @@ -419,6 +421,10 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, GT_FIFO_CTL_RC6_POLICY_STALL); } + /* Enable Decoupled MMIO only on BXT C stepping onwards */ + if (!IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER)) + info->has_decoupled_mmio = false; + intel_uncore_forcewake_reset(dev_priv, restore_forcewake); } @@ -641,6 +647,8 @@ intel_fw_table_check(struct drm_i915_private *dev_priv) num_ranges = dev_priv->uncore.fw_domains_table_entries; for (i = 0, prev = -1; i < num_ranges; i++, ranges++) { + WARN_ON_ONCE(IS_GEN9(dev_priv) && + (prev + 1) != (s32)ranges->start); WARN_ON_ONCE(prev >= (s32)ranges->start); prev = ranges->start; WARN_ON_ONCE(prev >= (s32)ranges->end); @@ -783,7 +791,7 @@ static const struct intel_forcewake_range __gen9_fw_ranges[] = { GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), - GEN_FW_RANGE(0xb480, 0xbfff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), @@ -831,6 +839,66 @@ unclaimed_reg_debug(struct drm_i915_private *dev_priv, __unclaimed_reg_debug(dev_priv, reg, read, before); } +static const enum decoupled_power_domain fw2dpd_domain[] = { + GEN9_DECOUPLED_PD_RENDER, + GEN9_DECOUPLED_PD_BLITTER, + GEN9_DECOUPLED_PD_ALL, + GEN9_DECOUPLED_PD_MEDIA, + GEN9_DECOUPLED_PD_ALL, + GEN9_DECOUPLED_PD_ALL, + GEN9_DECOUPLED_PD_ALL +}; + +/* + * Decoupled MMIO access for only 1 DWORD + */ +static void __gen9_decoupled_mmio_access(struct drm_i915_private *dev_priv, + u32 reg, + enum forcewake_domains fw_domain, + enum decoupled_ops operation) +{ + enum decoupled_power_domain dp_domain; + u32 ctrl_reg_data = 0; + + dp_domain = fw2dpd_domain[fw_domain - 1]; + + ctrl_reg_data |= reg; + ctrl_reg_data |= (operation << GEN9_DECOUPLED_OP_SHIFT); + ctrl_reg_data |= (dp_domain << GEN9_DECOUPLED_PD_SHIFT); + ctrl_reg_data |= GEN9_DECOUPLED_DW1_GO; + __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW1, ctrl_reg_data); + + if (wait_for_atomic((__raw_i915_read32(dev_priv, + GEN9_DECOUPLED_REG0_DW1) & + GEN9_DECOUPLED_DW1_GO) == 0, + FORCEWAKE_ACK_TIMEOUT_MS)) + DRM_ERROR("Decoupled MMIO wait timed out\n"); +} + +static inline u32 +__gen9_decoupled_mmio_read32(struct drm_i915_private *dev_priv, + u32 reg, + enum forcewake_domains fw_domain) +{ + __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain, + GEN9_DECOUPLED_OP_READ); + + return __raw_i915_read32(dev_priv, GEN9_DECOUPLED_REG0_DW0); +} + +static inline void +__gen9_decoupled_mmio_write(struct drm_i915_private *dev_priv, + u32 reg, u32 data, + enum forcewake_domains fw_domain) +{ + + __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW0, data); + + __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain, + GEN9_DECOUPLED_OP_WRITE); +} + + #define GEN2_READ_HEADER(x) \ u##x val = 0; \ assert_rpm_wakelock_held(dev_priv); @@ -935,6 +1003,28 @@ fwtable_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { GEN6_READ_FOOTER; \ } +#define __gen9_decoupled_read(x) \ +static u##x \ +gen9_decoupled_read##x(struct drm_i915_private *dev_priv, \ + i915_reg_t reg, bool trace) { \ + enum forcewake_domains fw_engine; \ + GEN6_READ_HEADER(x); \ + fw_engine = __fwtable_reg_read_fw_domains(offset); \ + if (fw_engine & ~dev_priv->uncore.fw_domains_active) { \ + unsigned i; \ + u32 *ptr_data = (u32 *) &val; \ + for (i = 0; i < x/32; i++, offset += sizeof(u32), ptr_data++) \ + *ptr_data = __gen9_decoupled_mmio_read32(dev_priv, \ + offset, \ + fw_engine); \ + } else { \ + val = __raw_i915_read##x(dev_priv, reg); \ + } \ + GEN6_READ_FOOTER; \ +} + +__gen9_decoupled_read(32) +__gen9_decoupled_read(64) __fwtable_read(8) __fwtable_read(16) __fwtable_read(32) @@ -1064,6 +1154,25 @@ fwtable_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bo GEN6_WRITE_FOOTER; \ } +#define __gen9_decoupled_write(x) \ +static void \ +gen9_decoupled_write##x(struct drm_i915_private *dev_priv, \ + i915_reg_t reg, u##x val, \ + bool trace) { \ + enum forcewake_domains fw_engine; \ + GEN6_WRITE_HEADER; \ + fw_engine = __fwtable_reg_write_fw_domains(offset); \ + if (fw_engine & ~dev_priv->uncore.fw_domains_active) \ + __gen9_decoupled_mmio_write(dev_priv, \ + offset, \ + val, \ + fw_engine); \ + else \ + __raw_i915_write##x(dev_priv, reg, val); \ + GEN6_WRITE_FOOTER; \ +} + +__gen9_decoupled_write(32) __fwtable_write(8) __fwtable_write(16) __fwtable_write(32) @@ -1287,6 +1396,14 @@ void intel_uncore_init(struct drm_i915_private *dev_priv) ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(fwtable); ASSIGN_READ_MMIO_VFUNCS(fwtable); + if (HAS_DECOUPLED_MMIO(dev_priv)) { + dev_priv->uncore.funcs.mmio_readl = + gen9_decoupled_read32; + dev_priv->uncore.funcs.mmio_readq = + gen9_decoupled_read64; + dev_priv->uncore.funcs.mmio_writel = + gen9_decoupled_write32; + } break; case 8: if (IS_CHERRYVIEW(dev_priv)) { @@ -1368,7 +1485,7 @@ int i915_reg_read_ioctl(struct drm_device *dev, for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) { if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) && - (INTEL_INFO(dev)->gen_mask & entry->gen_bitmask)) + (INTEL_INFO(dev_priv)->gen_mask & entry->gen_bitmask)) break; } diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h index 68db9621f1f0..8886cab19f98 100644 --- a/drivers/gpu/drm/i915/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/intel_vbt_defs.h @@ -280,7 +280,8 @@ struct common_child_dev_config { u8 dp_support:1; u8 tmds_support:1; u8 support_reserved:5; - u8 not_common3[12]; + u8 aux_channel; + u8 not_common3[11]; u8 iboost_level; } __packed; |
